Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/content/probe.ts

@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto'
+import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js'
+import { isPar2, parsePar2, type Par2Set } from '../par2/index.js'
+import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js'
+import { extractEntry } from '../rar/extract.js'
+import { walkRar } from '../rar/walk.js'
+import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js'
+import { listSevenZip } from '../sevenzip/list.js'
+import { sniffContentType, type ContentType } from './sniff.js'
+import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js'
+
+/**
+ * Content inspection's download-and-parse half (CONCEPT.md D21, D41),
+ * without a database: what the indexer needs to know about a file, from its
+ * segments. The indexer prepares a task from the database, the probe runs
+ * here or on a node (articles through the seed API's relay), and the
+ * indexer applies the result. Tasks and results are plain JSON.
+ */
+
+export interface TaskSegment {
+  number: number
+  id: string
+  /** Article size in bytes. */
+  size?: number
+}
+
+/** Inspect one file: its first segment, and more where its kind needs it. */
+export interface InspectTask {
+  kind: 'inspect'
+  fileId: number
+  /** real_name, else filename. */
+  name: string
+  segments: TaskSegment[]
+  /** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */
+  sevenZip: { volumes: number; last: TaskSegment[] } | null
+}
+
+export interface InspectProbe {
+  /** The whole file's size, from the yEnc header. */
+  size: number
+  contentType: ContentType | null
+  md5_16k: string | null
+  /** The file name in the yEnc header. */
+  yencName: string
+  nicovideo?: NicovideoInfo | null
+  /** A whole small SFV (latin1) or NZB (utf8). */
+  text?: string | null
+  par2?: Par2Set | null
+  archive?: RarInfo | null
+  /** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */
+  archiveNzbs?: { entryName: string; xml: string }[]
+  archiveNzbEntries?: number
+}
+
+/** Walk a RAR set's headers for its complete listing (D40). */
+export interface WalkTask {
+  kind: 'walk'
+  releaseId: number
+  volumes: TaskSegment[][]
+  maxSegments: number
+}
+
+export interface WalkProbe {
+  entries: Omit<RarEntry, 'stored' | 'dataOffset'>[]
+  complete: boolean
+  stopped?: string
+  headersEncrypted: boolean
+  segments: number
+}
+
+export type ContentTask = InspectTask | WalkTask
+
+/** Small posted files (NZBs, SFVs) that are read completely. */
+const WHOLE_MAX_PARTS = 40
+const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+/** Par2 index files are small; recovery volumes can be large and are not needed. */
+const PAR2_MAX_PARTS = 20
+const FIRST_CHUNK = 16 * 1024
+
+/** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */
+export function chunkMd5(data: Buffer, fileSize: number): string | undefined {
+  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
+  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
+}
+
+/** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */
+export function decodeBody(body: Buffer): DecodedSegment {
+  try {
+    return decodeSegment(body)
+  } catch (error) {
+    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+    return decodeSegment(body, { lenientSize: true })
+  }
+}
+
+/** All of a (small) file whose first segment is already decoded; null when it is too big. */
+export async function readWhole(first: Buffer, segments: TaskSegment[], fetch: BodyFetcher): Promise<Buffer | null> {
+  const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0)
+  if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
+  if (segments.length === 1) return first
+  let assembled: YencFile | undefined
+  for (const part of segments) {
+    const segment = decodeSegment(await fetch(part.id), { lenientSize: true })
+    assembled ??= YencFile.fromSegment(segment)
+    assembled.add(segment)
+  }
+  return assembled!.getBuffer()
+}
+
+/** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */
+export async function readPar2(first: Buffer, segments: TaskSegment[], fetch: BodyFetcher): Promise<Par2Set | null> {
+  if (!isPar2(first)) return null
+  if (segments.length > PAR2_MAX_PARTS) return null
+  if (segments.length === 1) return parsePar2(first)
+  let assembled: YencFile | undefined
+  for (const part of segments) {
+    const segment = decodeSegment(await fetch(part.id))
+    assembled ??= YencFile.fromSegment(segment)
+    assembled.add(segment)
+  }
+  return parsePar2(assembled!.getBuffer())
+}
+
+export function isNzbEntry(entry: Pick<RarEntry, 'name' | 'encrypted' | 'directory'>): boolean {
+  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+}
+
+/** NZBs that can be read from a single, unencrypted archive. */
+function hasNzbEntries(info: RarInfo): boolean {
+  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+}
+
+export async function probeInspection(
+  task: InspectTask,
+  fetch: BodyFetcher,
+  warn: (message: string) => void = () => {}
+): Promise<InspectProbe> {
+  const segment = decodeBody(await fetch(task.segments[0].id))
+  const size = segment.meta.begin.size
+  const contentType = sniffContentType(segment.data)
+  const probe: InspectProbe = {
+    size,
+    contentType,
+    md5_16k: chunkMd5(segment.data, size) ?? null,
+    yencName: segment.meta.begin.name,
+  }
+  // An encrypted NZB, rclone crypt or yEncBin's container: nothing to read.
+  if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe
+  if (NICOVIDEO_FILE.test(task.name)) {
+    probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data))
+    return probe
+  }
+  if (/\.sfv$/i.test(task.name)) {
+    probe.text = (await readWhole(segment.data, task.segments, fetch))?.toString('latin1') ?? null
+    return probe
+  }
+  if (contentType === 'nzb') {
+    probe.text = (await readWhole(segment.data, task.segments, fetch))?.toString('utf8') ?? null
+    return probe
+  }
+  probe.par2 = await readPar2(segment.data, task.segments, fetch)
+  if (probe.par2) return probe
+  if (contentType === '7z') {
+    probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null
+    return probe
+  }
+  const info = parseRarHeaders(segment.data)
+  probe.archive = info
+  if (info && hasNzbEntries(info)) {
+    probe.archiveNzbs = []
+    const archive = await readWhole(segment.data, task.segments, fetch)
+    const archived = archive && parseRarHeaders(archive)
+    const entries = archived ? archived.entries.filter(isNzbEntry) : []
+    probe.archiveNzbEntries = entries.length
+    for (const entry of entries) {
+      const xml = await extractEntry(archive!, entry, undefined, warn)
+      if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') })
+    }
+  }
+  return probe
+}
+
+export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise<WalkProbe> {
+  const reader = new SegmentReader(task.volumes, task.maxSegments, fetch)
+  const walked = await walkRar(reader)
+  return {
+    entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry),
+    complete: walked.complete,
+    stopped: walked.stopped,
+    headersEncrypted: walked.headersEncrypted,
+    segments: reader.fetched,
+  }
+}

lib/rar/extract.ts

@@ -0,0 +1,48 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { storedEntryData, type RarEntry } from './parse.js'
+
+const run = promisify(execFile)
+
+/** Tools that print one archive entry to stdout, tried in order. */
+const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
+  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
+  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
+]
+
+/**
+ * The unpacked data of one entry of a complete (single-volume) archive:
+ * copied out when stored, else unpacked with bsdtar (libarchive, in the
+ * Docker image) or unrar. Null when that is not possible.
+ */
+export async function extractEntry(
+  archive: Buffer,
+  entry: RarEntry,
+  maxBytes = 64 * 1024 * 1024,
+  warn: (message: string) => void = () => {}
+): Promise<Buffer | null> {
+  const stored = storedEntryData(archive, entry)
+  if (stored) return stored
+  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
+  if (entry.size !== null && entry.size > maxBytes) return null
+
+  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
+  try {
+    const file = path.join(dir, 'archive.rar')
+    await writeFile(file, archive)
+    for (const [tool, args] of EXTRACTORS) {
+      try {
+        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
+        if (stdout.length > 0) return stdout
+      } catch (error: any) {
+        if (error.code !== 'ENOENT') warn(`${tool} could not extract "${entry.name}": ${error.message}`)
+      }
+    }
+    return null
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}

lib/rar/index.ts

@@ -4,3 +4,5 @@
  */
 export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
 export { walkRar, type VolumeReader, type RarWalk } from './walk.js'
+export { SegmentReader, type BodyFetcher } from './segment_reader.js'
+export { extractEntry } from './extract.js'

lib/rar/segment_reader.ts

@@ -0,0 +1,82 @@
+import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '../yenc/index.js'
+import type { VolumeReader } from './walk.js'
+
+/** Downloads one article body (on a pooled connection, or through the seed API's relay). */
+export type BodyFetcher = (messageId: string) => Promise<Buffer>
+
+/**
+ * Reads volumes (each a list of segment message ids, in order) a segment at
+ * a time, finding the segment for an offset from the part size (yEnc
+ * posters use one size per set) and correcting from the `=ypart` offsets
+ * when a guess is off. Stops reading after `maxSegments` downloads.
+ */
+export class SegmentReader implements VolumeReader {
+  fetched = 0
+  private partSize = 0
+  private readonly sizes = new Map<number, number>()
+  private readonly cache = new Map<string, DecodedSegment>()
+
+  constructor(
+    private readonly files: { id: string }[][],
+    private readonly maxSegments: number,
+    private readonly fetchBody: BodyFetcher
+  ) {}
+
+  get volumes() {
+    return this.files.length
+  }
+
+  size(v: number) {
+    return this.sizes.get(v) ?? null
+  }
+
+  async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
+    const count = this.files[v].length
+    let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
+    for (let tries = 0; tries < 4; tries++) {
+      const segment = await this.segment(v, index)
+      if (!segment) return null
+      const end = segment.offset + segment.data.length
+      if (offset >= segment.offset && offset < end) {
+        const data = segment.data.subarray(offset - segment.offset)
+        if (!more || index + 1 >= count) return data
+        const next = await this.segment(v, index + 1)
+        return next ? Buffer.concat([data, next.data]) : null
+      }
+      const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
+      const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
+      if (moved === index) return null
+      index = moved
+    }
+    return null
+  }
+
+  private async segment(v: number, index: number): Promise<DecodedSegment | null> {
+    const key = `${v}:${index}`
+    const cached = this.cache.get(key)
+    if (cached) return cached
+    const part = this.files[v][index]
+    if (!part?.id || this.fetched >= this.maxSegments) return null
+    this.fetched++
+    let body: Buffer
+    try {
+      body = await this.fetchBody(part.id)
+    } catch {
+      // A missing article (or the relay failing) ends the walk with what was found.
+      return null
+    }
+    let segment: DecodedSegment
+    try {
+      segment = decodeSegment(body)
+    } catch (error) {
+      if (!(error instanceof YencError) || error instanceof YencCrcError) return null
+      segment = decodeSegment(body, { lenientSize: true })
+    }
+    this.sizes.set(v, segment.meta.begin.size)
+    if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
+    // Walks move forward: the last few segments are enough.
+    if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
+    this.cache.set(key, segment)
+    return segment
+  }
+}

lib/sevenzip/list.ts

@@ -0,0 +1,171 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, open, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { decodeSegment, type DecodedSegment } from '../yenc/index.js'
+import type { RarInfo } from '../rar/parse.js'
+import type { BodyFetcher } from '../rar/segment_reader.js'
+import {
+  checkHeader,
+  encodedHeaderEncrypted,
+  listingStart,
+  parseBsdtarListing,
+  parseSevenZipStart,
+  type SevenZipStart,
+} from './index.js'
+
+const run = promisify(execFile)
+
+/** The tail read for a listing: header plus packed header; more is not a header. */
+const MAX_TAIL_BYTES = 16 * 1024 * 1024
+
+/**
+ * Lists a 7z archive, also a split one (.7z.001, .002, …), from the first
+ * segment of the first volume (`firstData`, decoded) and the last segment(s)
+ * of the last one (CONCEPT.md D19). Those bytes are put at their place in a
+ * sparse file the size of the whole archive, which bsdtar (libarchive)
+ * lists: it only reads the start and the header at the end. Null when it
+ * isn't a 7z start, or the tail doesn't hold the header its start points to.
+ */
+export async function listSevenZip(
+  firstData: Buffer,
+  volumeCount: number,
+  lastVolume: { number: number; id: string }[],
+  fetch: BodyFetcher,
+  warn: (message: string) => void = () => {}
+): Promise<RarInfo | null> {
+  const start = parseSevenZipStart(firstData)
+  if (!start) return null
+  const split = volumeCount > 1
+  if (start.headerSize === 0) return sevenZipInfo([], false, false, split)
+
+  const tail = await readTail(lastVolume, start, fetch)
+  if (!tail) return null
+  // Encrypted headers (-mhe): known from the header alone, no listing possible (most 7z uploads).
+  if ('headersEncrypted' in tail) return sevenZipInfo([], true, true, split)
+
+  const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
+  try {
+    const archive = path.join(dir, 'archive.7z')
+    const handle = await open(archive, 'w')
+    try {
+      await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
+      await handle.write(tail.data, 0, tail.data.length, tail.from)
+      await handle.truncate(start.archiveSize)
+    } finally {
+      await handle.close()
+    }
+    return await bsdtar(archive, split, warn)
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}
+
+/** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
+async function bsdtar(archive: string, split: boolean, warn: (message: string) => void): Promise<RarInfo | null> {
+  let stdout = ''
+  try {
+    stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
+  } catch (error: any) {
+    if (/header is encrypted/i.test(String(error.stderr))) return sevenZipInfo([], true, true, split)
+    warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
+    return null
+  }
+  const entries = parseBsdtarListing(stdout)
+  const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
+  let encrypted = false
+  if (sample) {
+    try {
+      await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
+    } catch (error: any) {
+      // Without the data (it's all zeros here), anything else fails too: only this message counts.
+      encrypted = /content is encrypted/i.test(String(error.stderr))
+    }
+  }
+  return sevenZipInfo(entries, false, encrypted, split)
+}
+
+export function sevenZipInfo(
+  entries: { name: string; size: number; directory: boolean }[],
+  headersEncrypted: boolean,
+  encrypted: boolean,
+  split: boolean
+): RarInfo {
+  return {
+    format: '7z',
+    volume: split,
+    firstVolume: true,
+    solid: false,
+    headersEncrypted,
+    truncated: false,
+    next: 0,
+    entries: entries.map((e) => ({
+      name: e.name,
+      size: e.size,
+      packedSize: 0,
+      directory: e.directory,
+      encrypted: encrypted && !e.directory,
+      splitBefore: false,
+      splitAfter: false,
+      stored: false,
+      dataOffset: 0,
+    })),
+  }
+}
+
+/**
+ * The archive's tail from the last volume: the header (checked against its
+ * CRC) and, for an encoded header, the packed stream before it. `from` is
+ * its position in the whole archive.
+ */
+async function readTail(
+  last: { number: number; id: string }[],
+  start: SevenZipStart,
+  fetch: BodyFetcher
+): Promise<{ from: number; data: Buffer } | { headersEncrypted: true } | null> {
+  const ids = new Map(last.map((s) => [s.number, s]))
+  const numbers = [...ids.keys()]
+  if (numbers.length === 0) return null
+  const segments = new Map<number, DecodedSegment>()
+  const segment = async (n: number) => {
+    if (!segments.has(n)) {
+      const id = ids.get(n)?.id
+      if (!id) throw new Error(`part ${n} of the last 7z volume is missing`)
+      segments.set(n, decodeSegment(await fetch(id), { lenientSize: true }))
+    }
+    return segments.get(n)!
+  }
+
+  const lastNumber = numbers[numbers.length - 1]
+  const lastSegment = await segment(lastNumber)
+  const volumeSize = lastSegment.meta.begin.size
+  const volumeStart = start.archiveSize - volumeSize
+  const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
+  const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
+  if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
+
+  // Bytes [from, to) of the whole archive, from the last volume's parts.
+  const read = async (from: number, to: number): Promise<Buffer | null> => {
+    if (from < volumeStart) return null
+    const a = from - volumeStart
+    const b = to - volumeStart
+    const pieces: Buffer[] = []
+    for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
+      const s = await segment(n)
+      const offset = (s.meta.part?.begin ?? 1) - 1
+      if (offset !== (n - 1) * partSize) return null
+      pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
+    }
+    const data = Buffer.concat(pieces)
+    return data.length === b - a ? data : null
+  }
+
+  const header = await read(start.headerStart, start.archiveSize)
+  if (!header || !checkHeader(start, header)) return null
+  if (encodedHeaderEncrypted(header)) return { headersEncrypted: true }
+  const from = Math.min(listingStart(start, header), start.headerStart)
+  if (start.archiveSize - from > MAX_TAIL_BYTES) return null
+  const data = await read(from, start.archiveSize)
+  return data ? { from, data } : null
+}