Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/content/probe.ts

@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto'
+import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js'
+import { isPar2, parsePar2, type Par2Set } from '../par2/index.js'
+import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js'
+import { extractEntry } from '../rar/extract.js'
+import { walkRar } from '../rar/walk.js'
+import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js'
+import { listSevenZip } from '../sevenzip/list.js'
+import { sniffContentType, type ContentType } from './sniff.js'
+import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js'
+
+/**
+ * Content inspection's download-and-parse half (CONCEPT.md D21, D41),
+ * without a database: what the indexer needs to know about a file, from its
+ * segments. The indexer prepares a task from the database, the probe runs
+ * here or on a node (articles through the seed API's relay), and the
+ * indexer applies the result. Tasks and results are plain JSON.
+ */
+
+export interface TaskSegment {
+  number: number
+  id: string
+  /** Article size in bytes. */
+  size?: number
+}
+
+/** Inspect one file: its first segment, and more where its kind needs it. */
+export interface InspectTask {
+  kind: 'inspect'
+  fileId: number
+  /** real_name, else filename. */
+  name: string
+  segments: TaskSegment[]
+  /** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */
+  sevenZip: { volumes: number; last: TaskSegment[] } | null
+}
+
+export interface InspectProbe {
+  /** The whole file's size, from the yEnc header. */
+  size: number
+  contentType: ContentType | null
+  md5_16k: string | null
+  /** The file name in the yEnc header. */
+  yencName: string
+  nicovideo?: NicovideoInfo | null
+  /** A whole small SFV (latin1) or NZB (utf8). */
+  text?: string | null
+  par2?: Par2Set | null
+  archive?: RarInfo | null
+  /** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */
+  archiveNzbs?: { entryName: string; xml: string }[]
+  archiveNzbEntries?: number
+}
+
+/** Walk a RAR set's headers for its complete listing (D40). */
+export interface WalkTask {
+  kind: 'walk'
+  releaseId: number
+  volumes: TaskSegment[][]
+  maxSegments: number
+}
+
+export interface WalkProbe {
+  entries: Omit<RarEntry, 'stored' | 'dataOffset'>[]
+  complete: boolean
+  stopped?: string
+  headersEncrypted: boolean
+  segments: number
+}
+
+export type ContentTask = InspectTask | WalkTask
+
+/** Small posted files (NZBs, SFVs) that are read completely. */
+const WHOLE_MAX_PARTS = 40
+const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+/** Par2 index files are small; recovery volumes can be large and are not needed. */
+const PAR2_MAX_PARTS = 20
+const FIRST_CHUNK = 16 * 1024
+
+/** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */
+export function chunkMd5(data: Buffer, fileSize: number): string | undefined {
+  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
+  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
+}
+
+/** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */
+export function decodeBody(body: Buffer): DecodedSegment {
+  try {
+    return decodeSegment(body)
+  } catch (error) {
+    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+    return decodeSegment(body, { lenientSize: true })
+  }
+}
+
+/** All of a (small) file whose first segment is already decoded; null when it is too big. */
+export async function readWhole(first: Buffer, segments: TaskSegment[], fetch: BodyFetcher): Promise<Buffer | null> {
+  const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0)
+  if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
+  if (segments.length === 1) return first
+  let assembled: YencFile | undefined
+  for (const part of segments) {
+    const segment = decodeSegment(await fetch(part.id), { lenientSize: true })
+    assembled ??= YencFile.fromSegment(segment)
+    assembled.add(segment)
+  }
+  return assembled!.getBuffer()
+}
+
+/** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */
+export async function readPar2(first: Buffer, segments: TaskSegment[], fetch: BodyFetcher): Promise<Par2Set | null> {
+  if (!isPar2(first)) return null
+  if (segments.length > PAR2_MAX_PARTS) return null
+  if (segments.length === 1) return parsePar2(first)
+  let assembled: YencFile | undefined
+  for (const part of segments) {
+    const segment = decodeSegment(await fetch(part.id))
+    assembled ??= YencFile.fromSegment(segment)
+    assembled.add(segment)
+  }
+  return parsePar2(assembled!.getBuffer())
+}
+
+export function isNzbEntry(entry: Pick<RarEntry, 'name' | 'encrypted' | 'directory'>): boolean {
+  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+}
+
+/** NZBs that can be read from a single, unencrypted archive. */
+function hasNzbEntries(info: RarInfo): boolean {
+  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+}
+
+export async function probeInspection(
+  task: InspectTask,
+  fetch: BodyFetcher,
+  warn: (message: string) => void = () => {}
+): Promise<InspectProbe> {
+  const segment = decodeBody(await fetch(task.segments[0].id))
+  const size = segment.meta.begin.size
+  const contentType = sniffContentType(segment.data)
+  const probe: InspectProbe = {
+    size,
+    contentType,
+    md5_16k: chunkMd5(segment.data, size) ?? null,
+    yencName: segment.meta.begin.name,
+  }
+  // An encrypted NZB, rclone crypt or yEncBin's container: nothing to read.
+  if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe
+  if (NICOVIDEO_FILE.test(task.name)) {
+    probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data))
+    return probe
+  }
+  if (/\.sfv$/i.test(task.name)) {
+    probe.text = (await readWhole(segment.data, task.segments, fetch))?.toString('latin1') ?? null
+    return probe
+  }
+  if (contentType === 'nzb') {
+    probe.text = (await readWhole(segment.data, task.segments, fetch))?.toString('utf8') ?? null
+    return probe
+  }
+  probe.par2 = await readPar2(segment.data, task.segments, fetch)
+  if (probe.par2) return probe
+  if (contentType === '7z') {
+    probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null
+    return probe
+  }
+  const info = parseRarHeaders(segment.data)
+  probe.archive = info
+  if (info && hasNzbEntries(info)) {
+    probe.archiveNzbs = []
+    const archive = await readWhole(segment.data, task.segments, fetch)
+    const archived = archive && parseRarHeaders(archive)
+    const entries = archived ? archived.entries.filter(isNzbEntry) : []
+    probe.archiveNzbEntries = entries.length
+    for (const entry of entries) {
+      const xml = await extractEntry(archive!, entry, undefined, warn)
+      if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') })
+    }
+  }
+  return probe
+}
+
+export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise<WalkProbe> {
+  const reader = new SegmentReader(task.volumes, task.maxSegments, fetch)
+  const walked = await walkRar(reader)
+  return {
+    entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry),
+    complete: walked.complete,
+    stopped: walked.stopped,
+    headersEncrypted: walked.headersEncrypted,
+    segments: reader.fetched,
+  }
+}