Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/rar/extract.ts

@@ -0,0 +1,48 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import { storedEntryData, type RarEntry } from './parse.js'
+
+const run = promisify(execFile)
+
+/** Tools that print one archive entry to stdout, tried in order. */
+const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
+  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
+  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
+]
+
+/**
+ * The unpacked data of one entry of a complete (single-volume) archive:
+ * copied out when stored, else unpacked with bsdtar (libarchive, in the
+ * Docker image) or unrar. Null when that is not possible.
+ */
+export async function extractEntry(
+  archive: Buffer,
+  entry: RarEntry,
+  maxBytes = 64 * 1024 * 1024,
+  warn: (message: string) => void = () => {}
+): Promise<Buffer | null> {
+  const stored = storedEntryData(archive, entry)
+  if (stored) return stored
+  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
+  if (entry.size !== null && entry.size > maxBytes) return null
+
+  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
+  try {
+    const file = path.join(dir, 'archive.rar')
+    await writeFile(file, archive)
+    for (const [tool, args] of EXTRACTORS) {
+      try {
+        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
+        if (stdout.length > 0) return stdout
+      } catch (error: any) {
+        if (error.code !== 'ENOENT') warn(`${tool} could not extract "${entry.name}": ${error.message}`)
+      }
+    }
+    return null
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}

lib/rar/index.ts

@@ -4,3 +4,5 @@
  */
 export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
 export { walkRar, type VolumeReader, type RarWalk } from './walk.js'
+export { SegmentReader, type BodyFetcher } from './segment_reader.js'
+export { extractEntry } from './extract.js'

lib/rar/segment_reader.ts

@@ -0,0 +1,82 @@
+import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '../yenc/index.js'
+import type { VolumeReader } from './walk.js'
+
+/** Downloads one article body (on a pooled connection, or through the seed API's relay). */
+export type BodyFetcher = (messageId: string) => Promise<Buffer>
+
+/**
+ * Reads volumes (each a list of segment message ids, in order) a segment at
+ * a time, finding the segment for an offset from the part size (yEnc
+ * posters use one size per set) and correcting from the `=ypart` offsets
+ * when a guess is off. Stops reading after `maxSegments` downloads.
+ */
+export class SegmentReader implements VolumeReader {
+  fetched = 0
+  private partSize = 0
+  private readonly sizes = new Map<number, number>()
+  private readonly cache = new Map<string, DecodedSegment>()
+
+  constructor(
+    private readonly files: { id: string }[][],
+    private readonly maxSegments: number,
+    private readonly fetchBody: BodyFetcher
+  ) {}
+
+  get volumes() {
+    return this.files.length
+  }
+
+  size(v: number) {
+    return this.sizes.get(v) ?? null
+  }
+
+  async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
+    const count = this.files[v].length
+    let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
+    for (let tries = 0; tries < 4; tries++) {
+      const segment = await this.segment(v, index)
+      if (!segment) return null
+      const end = segment.offset + segment.data.length
+      if (offset >= segment.offset && offset < end) {
+        const data = segment.data.subarray(offset - segment.offset)
+        if (!more || index + 1 >= count) return data
+        const next = await this.segment(v, index + 1)
+        return next ? Buffer.concat([data, next.data]) : null
+      }
+      const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
+      const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
+      if (moved === index) return null
+      index = moved
+    }
+    return null
+  }
+
+  private async segment(v: number, index: number): Promise<DecodedSegment | null> {
+    const key = `${v}:${index}`
+    const cached = this.cache.get(key)
+    if (cached) return cached
+    const part = this.files[v][index]
+    if (!part?.id || this.fetched >= this.maxSegments) return null
+    this.fetched++
+    let body: Buffer
+    try {
+      body = await this.fetchBody(part.id)
+    } catch {
+      // A missing article (or the relay failing) ends the walk with what was found.
+      return null
+    }
+    let segment: DecodedSegment
+    try {
+      segment = decodeSegment(body)
+    } catch (error) {
+      if (!(error instanceof YencError) || error instanceof YencCrcError) return null
+      segment = decodeSegment(body, { lenientSize: true })
+    }
+    this.sizes.set(v, segment.meta.begin.size)
+    if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
+    // Walks move forward: the last few segments are enough.
+    if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
+    this.cache.set(key, segment)
+    return segment
+  }
+}