Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/ContentService.ts

@@ -1,24 +1,22 @@
-import { createHash } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
 import NntpService from '#services/NntpService'
 import Par2Service, { type Par2Result } from '#services/Par2Service'
-import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
+import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService'
 import NameService from '#services/NameService'
-import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
+import { decodeSegment } from '#lib/yenc/index'
 import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
 import {
   isArchiveStart,
   isLaterVolume,
   NICOVIDEO_FILE,
   NICOVIDEO_RELEASE,
-  nicovideoInfo,
   nicovideoReleaseName,
-  sniffContentType,
-  zstdPrefix,
   type ContentType,
   type NicovideoInfo,
 } from '#lib/content/index'
-import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
+import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe'
+import type { Segment } from '#lib/nzb/index'
 import { fileStem, releaseIdentity } from '#lib/release/index'
 import IngestService from '#services/IngestService'
 import { looksLikeFilename } from '#lib/subject/index'
@@ -28,8 +26,6 @@ import SfvService from '#services/SfvService'
 import ArchiveWalkService from '#services/ArchiveWalkService'
 import { segmentsOf } from '#services/FileStore'
 
-const FIRST_CHUNK = 16 * 1024
-
 /**
  * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
  * the file's first part. Undefined when that part is shorter than 16 KiB but
@@ -39,11 +35,6 @@ export function firstChunkMd5(body: Buffer, fileSize: number): string | undefine
   return chunkMd5(decodeSegment(body).data, fileSize)
 }
 
-function chunkMd5(data: Buffer, fileSize: number): string | undefined {
-  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
-  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
-}
-
 export interface ContentResult {
   par2?: Par2Result
   archive?: ArchiveResult
@@ -66,6 +57,8 @@ export interface ContentResult {
  * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
  * - par2 files are recognised by content and name every file of their set,
  * - RAR headers give the release grouping, name and contents.
+ * Inspection is prepare (database) → probe (downloads and parsing,
+ * lib/content/probe; here or on a seeding node, D41) → apply (database).
  */
 class ContentService {
   /** Random or missing names, and named first RAR volumes (for contents and password flag). */
@@ -128,89 +121,41 @@ class ContentService {
     return name
   }
 
-  /** Downloads a whole (small) file whose first segment is already decoded. */
-  private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
-    const parts = segmentsOf(file)
-    const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
-    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
-    if (parts.length === 1) return first
-
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const part of parts) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    return assembled!.getBuffer()
-  }
-
   /**
-   * Downloads a small archive that lists NZBs and imports each of them
-   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
-   * uploads by other posters.
+   * The database half before a probe (CONCEPT.md D41): what to inspect for
+   * a file, or null when there's nothing to do. A later 7z volume becomes its
+   * archive's first volume, once all volumes are in.
    */
-  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
-    const archive = await this.readWhole(first, file)
-    const info = archive && parseRarHeaders(archive)
-    if (!archive || !info) return []
-    const entries = info.entries.filter(isNzbEntry)
-    const results: NzbImportResult[] = []
-    for (const entry of entries) {
-      const xml = await extractEntry(archive, entry)
-      if (!xml) continue
-      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
-        entryName: entry.name,
-        moveContainer: entries.length === 1,
-      })
-      if (imported) results.push(imported)
-    }
-    return results
-  }
-
-  async inspect(fileId: number): Promise<ContentResult | null> {
+  async prepare(fileId: number): Promise<InspectTask | null> {
     const file = await db.from('files').where('id', fileId).first()
-    const binary = await db.from('binaries').where('file_id', fileId).first()
     if (!file || !this.isCandidate(file.filename)) return null
-    // A later 7z volume: list the set from its first volume once all volumes are in.
     if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
       const volumes = await SevenZipService.volumes(file)
       if (!volumes || volumes[0].archive) return null
-      return this.inspect(Number(volumes[0].id))
+      return this.prepare(Number(volumes[0].id))
     }
-    const first = segmentsOf(file)[0]
-    if (!first?.id) return null
+    const segments = segmentsOf(file)
+    if (!segments[0]?.id) return null
+    // A 7z listing reads the last volume's tail (sniffing decides whether it's 7z: random names too).
+    const volumes = await SevenZipService.volumes(file)
+    const last = volumes && volumes.length > 1 ? segmentsOf(volumes[volumes.length - 1]) : segments
+    return {
+      kind: 'inspect',
+      fileId: Number(file.id),
+      name: String(file.real_name ?? file.filename),
+      segments: taskSegments(segments),
+      sevenZip: volumes ? { volumes: volumes.length, last: taskSegments(last, true) } : null,
+    }
+  }
 
-    const conn = await NntpService.acquire()
-    let failed = false
-    let body: Buffer
-    try {
-      body = await NntpService.fetchBody(conn, first.id)
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
-    let segment
-    try {
-      segment = decodeSegment(body)
-    } catch (error) {
-      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
-      segment = decodeSegment(body, { lenientSize: true })
-    }
-    const size = segment.meta.begin.size
-    const contentType = sniffContentType(segment.data)
-    await db.from('files').where('id', fileId).update({ content_type: contentType })
+  /** The database half after a probe: names, groupings, listings and flags from what it found. */
+  async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
+    const result: ContentResult = { named: 0, contentType: probe.contentType }
+    const file = await db.from('files').where('id', task.fileId).first()
+    if (!file) return result
+    const binary = await db.from('binaries').where('file_id', task.fileId).first()
+    const { contentType } = probe
+    await db.from('files').where('id', file.id).update({ content_type: contentType })
 
     // Fingerprint, so par2 sets (now or later) can name this file exactly.
     if (binary) {
@@ -218,13 +163,12 @@ class ContentService {
         .from('binaries')
         .where('id', binary.id)
         .update({
-          size: db.raw('coalesce(size, ?)', [size]),
-          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
+          size: db.raw('coalesce(size, ?)', [probe.size]),
+          md5_16k: db.raw('coalesce(md5_16k, ?)', [probe.md5_16k]),
         })
     }
 
-    const result: ContentResult = { named: 0, contentType }
-    result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
+    result.yencName = await this.nameFromYenc(file, binary, probe.yencName ?? '')
     if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
       // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
       // to read, and useless without the key.
@@ -234,53 +178,105 @@ class ContentService {
       }
       return result
     }
-    const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
+    const nicovideo = NICOVIDEO_FILE.exec(task.name)
     if (nicovideo) {
-      const info = nicovideoInfo(await zstdPrefix(segment.data))
-      if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
+      if (probe.nicovideo) result.nicovideo = await this.nameNicovideo(file, probe.nicovideo, nicovideo[1])
       return result
     }
-    if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
-      const text = await this.readWhole(segment.data, file)
-      if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
+    if (/\.sfv$/i.test(task.name)) {
+      if (typeof probe.text === 'string') result.sfvMoved = (await SfvService.attach(file, probe.text)) ?? undefined
       return result
     }
     if (contentType === 'nzb') {
-      const xml = await this.readWhole(segment.data, file)
-      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
+      if (typeof probe.text === 'string') result.nzb = (await NzbImportService.import(file, probe.text)) ?? undefined
       return result
     }
-    const set = await Par2Service.read(segment.data, segmentsOf(file))
-    if (set && binary) {
-      result.par2 = await Par2Service.attach(binary, set)
-    } else if (contentType === '7z') {
-      const info = await SevenZipService.list(file, segment.data)
-      if (info) result.archive = await ArchiveService.record(file, binary, info)
-    } else {
-      const info = parseRarHeaders(segment.data)
-      if (info) {
-        result.archive = await ArchiveService.record(file, binary, info)
-        // The rest of the listing, later and at low priority (D40).
-        await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, info)
-        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
+    if (probe.par2 && binary) {
+      result.par2 = await Par2Service.attach(binary, probe.par2)
+    } else if (probe.archive && probe.archive.format === '7z') {
+      result.archive = await ArchiveService.record(file, binary, probe.archive)
+    } else if (probe.archive) {
+      result.archive = await ArchiveService.record(file, binary, probe.archive)
+      // The rest of the listing, later and at low priority (D40).
+      await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, probe.archive)
+      if (probe.archiveNzbs?.length) {
+        result.archiveNzbs = []
+        for (const { entryName, xml } of probe.archiveNzbs) {
+          const imported = await NzbImportService.import(file, xml, { entryName, moveContainer: (probe.archiveNzbEntries ?? 0) === 1 })
+          if (imported) result.archiveNzbs.push(imported)
+        }
       }
     }
     if (binary) result.named = await NameService.resolve([binary.id])
     return result
   }
+
+  /** Inspects a file here: prepare, probe on the inspection pool's connections, apply. */
+  async inspect(fileId: number): Promise<ContentResult | null> {
+    const task = await this.prepare(fileId)
+    if (!task) return null
+    const probe = await probeInspection(task, fetchPooled, (message) => logger.warn(message))
+    return this.apply(task, probe)
+  }
 }
 
-/** NZBs that can be read from a single, unencrypted archive. */
-function hasNzbEntries(info: RarInfo): boolean {
-  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+/** A task's segment list: all of a small file's; of a big one, the first and the last few (a 7z tail). */
+function taskSegments(segments: Segment[], tailOnly = false): TaskSegment[] {
+  const list = segments.map(({ number, id, size }) => ({ number, id, size }))
+  if (list.length <= TASK_SEGMENTS) return list
+  return tailOnly ? list.slice(-TASK_SEGMENTS) : [list[0], ...list.slice(-TASK_SEGMENTS)]
 }
 
-function isNzbEntry(entry: RarEntry): boolean {
-  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+/** More than whole files (40 parts) and par2 files (20) are read; a 7z tail of 16 MB fits easily. */
+const TASK_SEGMENTS = 40
+
+/** One article body on a pooled connection. */
+async function fetchPooled(messageId: string): Promise<Buffer> {
+  const conn = await NntpService.acquire()
+  let failed = false
+  try {
+    return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+  } catch (error: any) {
+    // A missing article (430) leaves the connection fine.
+    failed = !/^4\d\d/.test(String(error.message))
+    throw error
+  } finally {
+    NntpService.release(conn, failed)
+  }
 }
 
-/** Small posted files (NZBs) that are read completely. */
-const WHOLE_MAX_PARTS = 40
-const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+/**
+ * One line per finding, as worker:collection logs them (and the seed API
+ * for nodes' results).
+ */
+export function describeContent(filename: string, content: ContentResult | null): string[] {
+  const lines: string[] = []
+  if (content?.par2) {
+    const { par2 } = content
+    lines.push(`Par2 "${filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+  } else if (content?.archive) {
+    const { info, releaseRenamed, grouped } = content.archive
+    const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+      .filter(Boolean)
+      .join(', ')
+    lines.push(
+      `Archive "${filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+        (grouped ? ', grouped into a release' : '') +
+        (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+    )
+  }
+  if (content?.named) lines.push(`"${filename}" got its real name from par2.`)
+  if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
+  if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
+  if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
+  if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
+  if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
+  if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
+  if (content?.nzb) {
+    const n = content.nzb
+    lines.push(`NZB "${filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
+  }
+  return lines
+}
 
 export default new ContentService()