usenet-indexer/lib/content/probe.ts

import { createHash } from 'node:crypto'
import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js'
import { isPar2, parsePar2, type Par2Set } from '../par2/index.js'
import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js'
import { extractEntry } from '../rar/extract.js'
import { walkRar } from '../rar/walk.js'
import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js'
import { listSevenZip } from '../sevenzip/list.js'
import { sniffContentType, type ContentType } from './sniff.js'
import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js'

/**
 * Content inspection's download-and-parse half (CONCEPT.md D21, D41),
 * without a database: what the indexer needs to know about a file, from its
 * segments. The indexer prepares a task from the database, the probe runs
 * here or on a node (articles through the seed API's relay), and the
 * indexer applies the result. Tasks and results are plain JSON.
 */

export interface TaskSegment {
  number: number
  id: string
  /** Article size in bytes. */
  size?: number
}

/** Inspect one file: its first segment, and more where its kind needs it. */
export interface InspectTask {
  kind: 'inspect'
  fileId: number
  /** real_name, else filename. */
  name: string
  segments: TaskSegment[]
  /** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */
  sevenZip: { volumes: number; last: TaskSegment[] } | null
}

export interface InspectProbe {
  /** The whole file's size, from the yEnc header. */
  size: number
  contentType: ContentType | null
  md5_16k: string | null
  /** The file name in the yEnc header. */
  yencName: string
  nicovideo?: NicovideoInfo | null
  /** A whole small SFV (latin1) or NZB (utf8). */
  text?: string | null
  par2?: Par2Set | null
  archive?: RarInfo | null
  /** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */
  archiveNzbs?: { entryName: string; xml: string }[]
  archiveNzbEntries?: number
}

/** Walk a RAR set's headers for its complete listing (D40). */
export interface WalkTask {
  kind: 'walk'
  releaseId: number
  volumes: TaskSegment[][]
  maxSegments: number
}

export interface WalkProbe {
  entries: Omit<RarEntry, 'stored' | 'dataOffset'>[]
  complete: boolean
  stopped?: string
  headersEncrypted: boolean
  segments: number
}

export type ContentTask = InspectTask | WalkTask

/** Small posted files (NZBs, SFVs) that are read completely. */
const WHOLE_MAX_PARTS = 40
const WHOLE_MAX_BYTES = 25 * 1024 * 1024
/** Par2 index files are small; recovery volumes can be large and are not needed. */
const PAR2_MAX_PARTS = 20
const FIRST_CHUNK = 16 * 1024

/** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */
export function chunkMd5(data: Buffer, fileSize: number): string | undefined {
  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
}

/** A "no such article" answer: 430 from the news server, 410 from the seed relay. */
export function isMissingArticle(error: unknown): boolean {
  return /(^|[^0-9])(430|410)([^0-9]|$)/.test(String((error as any)?.message ?? error)) && !/timed out|no answer/i.test(String((error as any)?.message))
}

/** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */
export function decodeBody(body: Buffer): DecodedSegment {
  try {
    return decodeSegment(body)
  } catch (error) {
    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
    return decodeSegment(body, { lenientSize: true })
  }
}

/**
 * All of a (small) file whose first segment is already decoded; null when it is too big.
 * The first segment isn't downloaded again (it was, for every par2 set, NZB and SFV).
 */
export async function readWhole(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Buffer | null> {
  const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0)
  if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
  if (segments.length === 1) return first.data
  const assembled = YencFile.fromSegment(first)
  assembled.add(first)
  for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id), { lenientSize: true }))
  return assembled.getBuffer()
}

/** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */
export async function readPar2(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Par2Set | null> {
  if (!isPar2(first.data)) return null
  if (segments.length > PAR2_MAX_PARTS) return null
  if (segments.length === 1) return parsePar2(first.data)
  const assembled = YencFile.fromSegment(first)
  assembled.add(first)
  for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id)))
  return parsePar2(assembled.getBuffer())
}

export function isNzbEntry(entry: Pick<RarEntry, 'name' | 'encrypted' | 'directory'>): boolean {
  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
}

/** NZBs that can be read from a single, unencrypted archive. */
function hasNzbEntries(info: RarInfo): boolean {
  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
}

export async function probeInspection(
  task: InspectTask,
  fetch: BodyFetcher,
  warn: (message: string) => void = () => {}
): Promise<InspectProbe> {
  let body: Buffer
  try {
    body = await fetch(task.segments[0].id)
  } catch (error: any) {
    // The server no longer has it (430; 410 through the seed relay), typically a takedown. A final
    // answer, and an expensive one: Eweka takes ~450 ms per missing article, so it isn't retried
    // (LOG.md 2026-10-06 (14)).
    if (isMissingArticle(error)) return { size: 0, contentType: 'missing', md5_16k: null, yencName: '' }
    throw error
  }
  const segment = decodeBody(body)
  const size = segment.meta.begin.size
  const contentType = sniffContentType(segment.data)
  const probe: InspectProbe = {
    size,
    contentType,
    md5_16k: chunkMd5(segment.data, size) ?? null,
    yencName: segment.meta.begin.name,
  }
  // An encrypted NZB, rclone crypt or yEncBin's container: nothing to read.
  if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe
  if (NICOVIDEO_FILE.test(task.name)) {
    probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data))
    return probe
  }
  if (/\.sfv$/i.test(task.name)) {
    probe.text = (await readWhole(segment, task.segments, fetch))?.toString('latin1') ?? null
    return probe
  }
  if (contentType === 'nzb') {
    probe.text = (await readWhole(segment, task.segments, fetch))?.toString('utf8') ?? null
    return probe
  }
  probe.par2 = await readPar2(segment, task.segments, fetch)
  if (probe.par2) return probe
  if (contentType === '7z') {
    probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null
    return probe
  }
  const info = parseRarHeaders(segment.data)
  probe.archive = info
  if (info && hasNzbEntries(info)) {
    probe.archiveNzbs = []
    const archive = await readWhole(segment, task.segments, fetch)
    const archived = archive && parseRarHeaders(archive)
    const entries = archived ? archived.entries.filter(isNzbEntry) : []
    probe.archiveNzbEntries = entries.length
    for (const entry of entries) {
      const xml = await extractEntry(archive!, entry, undefined, warn)
      if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') })
    }
  }
  return probe
}

export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise<WalkProbe> {
  const reader = new SegmentReader(task.volumes, task.maxSegments, fetch)
  const walked = await walkRar(reader)
  return {
    entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry),
    complete: walked.complete,
    stopped: walked.stopped,
    headersEncrypted: walked.headersEncrypted,
    segments: reader.fetched,
  }
}