Rebuild: phase 2's prober reads the probe queue

probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims
waiting candidates of settled releases (lone, then par2, then archive),
fetches each article with BODY over connections of its own (no
compression) and inspects it: the yEnc name and size, the content type
from the first bytes, a par2 set's files, a RAR volume's entries and
whether they're encrypted. Results per release and kind in
release_inspections (CHUNK_TABLESPACE, the HDD); on the release
passworded, inner_name (the largest content name) and inspected_at.
430 = missing, not yEnc = error, both final; connection trouble puts
the candidate back.

Fetch and parse in one process: an article takes ~0.1 s per connection,
parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc,
lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body;
yencode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/prober.ts

@@ -0,0 +1,183 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { setTimeout as sleep } from 'node:timers/promises'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { MissingArticle, XoverClient } from '#lib/nntp/index'
+import { YencError } from '#lib/yenc/index'
+import { inspect, innerName, type Inspection } from '#lib/probe/index'
+
+interface Candidate {
+  id: number
+  releaseId: number
+  kind: string
+  messageId: string
+}
+
+type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection }
+
+/** Kinds in the order they're probed: a random name's real name first, then par2 sets, then archives. */
+const KINDS = ['lone', 'par2', 'archive']
+const FLUSH_MS = 2000
+const REPORT_MS = 60_000
+
+/**
+ * Phase 2's first step (REBUILD.md, part 6): reads the probe queue's
+ * articles (settled releases first, by kind) over PROBE_CONNECTIONS
+ * connections of its own, without compression, and stores what they show:
+ * per release and kind an Inspection, and on the release whether it's
+ * passworded and its inner name. One process: fetching (~0.1 s an article)
+ * dwarfs parsing (a few ms), so there's no queue between them.
+ */
+export class Prober {
+  private queue: Candidate[] = []
+  private outcomes: Outcome[] = []
+  private refilling: Promise<void> | null = null
+  private stopped = false
+  private stats = { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, since: Date.now() }
+
+  async run() {
+    // A previous run's claims: one prober at a time.
+    const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
+    if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
+    const connections = coordinatorConfig.probe.connections
+    logger.info(`Probing with ${connections} connections.`)
+    const flusher = (async () => {
+      while (!this.stopped || this.outcomes.length > 0) {
+        await sleep(FLUSH_MS)
+        await this.flush().catch((error) => logger.error(`Probing: storing results failed: ${error.message}`))
+        this.report()
+      }
+    })()
+    await Promise.all(Array.from({ length: connections }, () => this.lane()))
+    // What was claimed and not probed goes back.
+    for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
+    this.stopped = true
+    await flusher
+  }
+
+  stop() {
+    this.stopped = true
+  }
+
+  /** One connection's loop: an article at a time, reconnecting after a failure. */
+  private async lane() {
+    let client: XoverClient | null = null
+    while (!this.stopped) {
+      const candidate = await this.next()
+      if (!candidate) {
+        await sleep(30_000)
+        continue
+      }
+      try {
+        client ??= await XoverClient.connect({
+          host: env.get('NNTP_HOST')!,
+          port: env.get('NNTP_PORT', 563),
+          user: env.get('NNTP_USER'),
+          password: env.get('NNTP_PASSWORD'),
+          secure: env.get('NNTP_SECURE', true),
+          compress: false,
+          timeoutMs: coordinatorConfig.nntpTimeoutMs,
+        })
+        const body = await client.body(candidate.messageId)
+        this.stats.articles++
+        this.stats.bytes += body.length
+        try {
+          this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
+        } catch (error: any) {
+          // Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
+          if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
+          this.outcomes.push({ candidate, status: 'error' })
+        }
+      } catch (error: any) {
+        if (error instanceof MissingArticle) {
+          // Taken down or expired: final (Eweka takes ~450 ms per missing article).
+          this.stats.articles++
+          this.outcomes.push({ candidate, status: 'missing' })
+          continue
+        }
+        logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
+        this.outcomes.push({ candidate, status: 'waiting' })
+        client?.close()
+        client = null
+        await sleep(5000)
+      }
+    }
+    client?.close()
+  }
+
+  private async next(): Promise<Candidate | undefined> {
+    if (this.queue.length < coordinatorConfig.probe.batch / 2) {
+      this.refilling ??= this.claim().finally(() => (this.refilling = null))
+      if (this.queue.length === 0) await this.refilling
+    }
+    return this.queue.shift()
+  }
+
+  /** Waiting candidates of settled releases, by kind, claimed so another round doesn't take them. */
+  private async claim() {
+    for (const kind of KINDS) {
+      const want = coordinatorConfig.probe.batch - this.queue.length
+      if (want <= 0) return
+      const { rows } = await db.rawQuery(
+        `UPDATE probe_candidates SET status = 'taken' WHERE id IN (
+           SELECT pc.id FROM probe_candidates pc JOIN releases r ON r.id = pc.release_id
+           WHERE pc.status = 'waiting' AND pc.kind = ? AND r.settled_at IS NOT NULL
+           ORDER BY pc.id LIMIT ? FOR UPDATE OF pc SKIP LOCKED)
+         RETURNING id, release_id, kind, message_id`,
+        [kind, want]
+      )
+      for (const r of rows) this.queue.push({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id })
+    }
+  }
+
+  /** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
+  private async flush() {
+    const outcomes = this.outcomes.splice(0)
+    if (outcomes.length === 0) return
+    for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
+    await db.transaction(async (trx) => {
+      await trx.rawQuery(
+        `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
+        [JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
+      )
+      const done = outcomes.filter((o) => o.inspection)
+      if (done.length === 0) return
+      await trx.rawQuery(
+        `INSERT INTO release_inspections (release_id, kind, result)
+         SELECT v.release_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind text, result jsonb)
+         ON CONFLICT (release_id, kind) DO UPDATE SET result = excluded.result, created_at = now()`,
+        [JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, kind: o.candidate.kind, result: o.inspection })))]
+      )
+      // Each touched release from all its inspections (earlier rounds may have stored its other kinds).
+      const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
+      const { rows } = await trx.rawQuery(`SELECT release_id, kind, result FROM release_inspections WHERE release_id = ANY(?)`, [ids])
+      const byRelease = new Map<number, Record<string, Inspection>>()
+      for (const r of rows) {
+        const id = Number(r.release_id)
+        byRelease.set(id, { ...byRelease.get(id), [r.kind]: r.result })
+      }
+      const updates = [...byRelease].map(([id, all]) => ({
+        id,
+        passworded: Object.values(all).some((i) => i.passworded === true),
+        inner_name: innerName(all),
+      }))
+      await trx.rawQuery(
+        `UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
+         FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
+        [JSON.stringify(updates)]
+      )
+    })
+  }
+
+  private report() {
+    const seconds = (Date.now() - this.stats.since) / 1000
+    if (seconds < REPORT_MS / 1000) return
+    const { articles, bytes, done, missing, error } = this.stats
+    logger.info(
+      `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
+        `${done} inspected, ${missing} missing, ${error} not yEnc.`
+    )
+    this.stats = { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, since: Date.now() }
+  }
+}

commands/probe_run.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Prober } from '#services/prober'
+
+/** Phase 2's prober (REBUILD.md, part 6): reads the probe queue's articles and stores what they show. */
+export default class ProbeRun extends BaseCommand {
+  static commandName = 'probe:run'
+  static description = "Reads the probe queue's articles (PROBE_CONNECTIONS) and stores what they show"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  async run() {
+    const prober = new Prober()
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      this.logger.info('Stopping: finishing the articles in flight.')
+      prober.stop()
+      await finished
+    })
+    await prober.run()
+    this.logger.info('Prober stopped.')
+    done!()
+    await this.terminate()
+  }
+}

config/coordinator.ts

@@ -65,6 +65,16 @@ export default {
    * under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
    */
   nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
+  probe: {
+    /**
+     * NNTP connections the prober reads articles with, besides the
+     * coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
+     * couple stay free for downloads. ~7 MB/s each (~10 articles/s).
+     */
+    connections: env.get('PROBE_CONNECTIONS', 6),
+    /** Candidates claimed at a time. */
+    batch: 200,
+  },
   spool: {
     dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
     /** Ranges kept fetched ahead, per active lease. */

database/migrations/1770000001700_release_inspections.ts

@@ -0,0 +1,38 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * Phase 2's probe results (REBUILD.md, part 6): per release and probe kind
+ * what its article showed (yEnc name, content type, a few names inside,
+ * encryption), in CHUNK_TABLESPACE (the HDD) when set: they're read per
+ * release, not searched. On the catalog only what search needs: whether the
+ * release is passworded and its best inner name.
+ */
+export default class extends BaseSchema {
+  async up() {
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    const where = tablespace ? ` TABLESPACE ${tablespace}` : ''
+    this.schema.raw(`
+      CREATE TABLE release_inspections (
+        release_id bigint NOT NULL,
+        kind varchar(8) NOT NULL,
+        result jsonb NOT NULL,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (release_id, kind)${tablespace ? ` USING INDEX TABLESPACE ${tablespace}` : ''}
+      )${where}`)
+    this.schema.alterTable('releases', (table) => {
+      table.boolean('passworded').nullable()
+      table.string('inner_name', 255).nullable()
+      table.timestamp('inspected_at', { useTz: true }).nullable()
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => {
+      table.dropColumn('passworded')
+      table.dropColumn('inner_name')
+      table.dropColumn('inspected_at')
+    })
+    this.schema.dropTable('release_inspections')
+  }
+}

deploy/README.md

@@ -17,6 +17,7 @@ homelab with the deploy key, whose forced command
 | `migrate` | `usenet-rebuild` (target `app`) | `migration:run`, before every deploy |
 | `web` | `usenet-rebuild` | node API, catalog API, Newznab, web interface; port 3334 |
 | `coordinator` | `usenet-rebuild` | prefetching (all NNTP connections), planning, the tail, merging, settling |
+| `prober` | `usenet-rebuild` | phase 2: reads the probe queue's articles over `PROBE_CONNECTIONS` (6) connections, stores what they show |
 | `edge` | `usenet-rebuild-edge` (target `edge`) | the Rust worker, `--threads $EDGE_THREADS` (8) |
 | `edge-ts` | `usenet-rebuild` | the TypeScript worker, profile `edge-ts`, not started |
 | `ace` | `usenet-rebuild` | one-off ace commands, profile `tools` |
@@ -62,6 +63,7 @@ docker compose up -d --no-deps edge coordinator
 - Stop `edge` before `web`: a worker holding a stream keeps the server from
   exiting.
 - The NNTP account is shared with voyager 1 and downloads: `NNTP_CONNECTIONS`
-  (40) plus one plain connection, out of 50.
+  (40) plus one plain connection, plus the prober's `PROBE_CONNECTIONS` (6),
+  out of 50.
 - `deploy.sh` doesn't update itself: after changing it, install it again
   (`install -m 755 src/deploy/deploy.sh ~/bin/deploy-usenet-rebuild`).

deploy/compose.yaml

@@ -82,6 +82,16 @@ services:
       <<: *app-env
       NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-40}
 
+  # Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS
+  # connections of its own (next to the coordinator's 40 + 1; the account has 50).
+  prober:
+    <<: *app
+    command: ['node', 'ace', 'probe:run']
+    volumes: []
+    environment:
+      <<: *app-env
+      PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-6}
+
   # The Rust worker; it has no NNTP account and talks to web only. Stopped
   # before web (a worker holding a stream keeps the server from exiting).
   edge:

deploy/deploy.sh

@@ -45,10 +45,10 @@ docker compose run --rm --no-deps migrate
 # stream keeps the web server from exiting.
 echo "== recreating app services"
 docker compose stop edge
-docker compose up -d --no-deps web coordinator
+docker compose up -d --no-deps web coordinator prober
 docker compose up -d --no-deps edge
 echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
 # The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
 docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null
-docker compose ps --format '{{.Service}} {{.Status}}' web coordinator edge
+docker compose ps --format '{{.Service}} {{.Status}}' web coordinator prober edge
 echo "== deployed $SHA"

lib/content/index.ts

@@ -0,0 +1,2 @@
+/** Content recognition by magic bytes (from voyager 1's lib/content). */
+export { sniffContentType, type ContentType } from './sniff.js'

lib/content/sniff.ts

@@ -0,0 +1,81 @@
+/**
+ * Recognises a file type from its first bytes, independent of AdonisJS.
+ * `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
+ * "nzbe", 16 random bytes (salt/IV), then ciphertext.
+ * `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com`
+ * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
+ */
+export type ContentType =
+  /** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */
+  | 'missing'
+  | 'nzbe'
+  | 'par2'
+  | 'rar4'
+  | 'rar5'
+  | '7z'
+  | 'zip'
+  | 'zstd'
+  | 'gzip'
+  | 'xz'
+  | 'matroska'
+  | 'mp4'
+  | 'rclone'
+  | 'yencbin'
+  | 'iso'
+  | 'udf'
+  | 'pdf'
+  | 'nzb'
+  | 'jpeg'
+  | 'png'
+  | 'mp3'
+  | 'flac'
+  | 'avi'
+  | 'wav'
+  | 'ogg'
+  | 'bzip2'
+
+/** Magic bytes at an offset; null matches any byte. */
+const SIGNATURES: [ContentType, number, (number | null)[]][] = [
+  // Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…):
+  // the bytes they share.
+  ['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]],
+  ['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
+  ['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
+  ['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
+  ['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]],
+  ['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]],
+  ['zip', 0, [0x50, 0x4b, 0x03, 0x04]],
+  ['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]],
+  ['gzip', 0, [0x1f, 0x8b]],
+  ['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]],
+  ['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]],
+  ['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box
+  ['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt
+  ['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]],
+  ['jpeg', 0, [0xff, 0xd8, 0xff]],
+  ['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]],
+  ['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag
+  ['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC"
+  ['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS"
+  ['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh"
+]
+
+export function sniffContentType(data: Buffer): ContentType | null {
+  for (const [type, offset, bytes] of SIGNATURES) {
+    if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type
+  }
+  // Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
+  // ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.
+  if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso'
+  if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' &&
+      /^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf'
+  // RIFF container: AVI or WAV by form type.
+  if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') {
+    const form = data.subarray(8, 12).toString('latin1')
+    if (form === 'AVI ') return 'avi'
+    if (form === 'WAVE') return 'wav'
+  }
+  const head = data.subarray(0, 512).toString('latin1')
+  if (/^\s*(<\?xml[^>]*>\s*)?(<!DOCTYPE nzb|<nzb[\s>])/.test(head)) return 'nzb'
+  return null
+}

lib/nntp/index.ts

@@ -1,4 +1,4 @@
-export { XoverClient, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
+export { XoverClient, MissingArticle, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
 export { parseXrefGroups } from './xref.js'
 export { parseGroupResponse, type GroupInfo } from './group.js'
 export { hedge, type Request, type Hedged } from './hedge.js'

lib/nntp/xover_client.ts

@@ -34,6 +34,9 @@ const DOT_LINE = Buffer.from('.\r\n')
  * followed by a plain "." line; an empty range is a 420 without body.
  * About six times less traffic per header (LOG.md 2026-10-05).
  */
+/** The server doesn't have the article (430): taken down or expired. */
+export class MissingArticle extends Error {}
+
 export class XoverClient {
   private chunks: Buffer[] = []
   private length = 0
@@ -228,6 +231,37 @@ export class XoverClient {
     }
   }
 
+  /**
+   * An article's body by message-id, raw (dot-stuffed, "\r\n.\r\n" cut off):
+   * yEnc decoding undoes the stuffing. A connection without COMPRESS GZIP.
+   * 430 (no such article) throws MissingArticle.
+   */
+  async body(messageId: string): Promise<Buffer> {
+    return this.timed(async () => {
+      const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
+      const reply = await this.command(`BODY ${id}`)
+      if (reply.startsWith('430') || reply.startsWith('423')) throw new MissingArticle(reply)
+      await this.expect(reply, ['222'])
+      return this.rawBody()
+    })
+  }
+
+  /** A multi-line body up to "\r\n.\r\n", as bytes, the terminator cut off. */
+  private async rawBody(): Promise<Buffer> {
+    let searched = 0
+    while (true) {
+      const all = this.buffer()
+      if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) {
+        this.take(3)
+        return Buffer.alloc(0)
+      }
+      const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
+      if (at !== -1) return Buffer.from(this.take(at + TERMINATOR.length).subarray(0, at + 2))
+      searched = all.length
+      await this.more()
+    }
+  }
+
   /** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */
   private async plainBody(): Promise<string> {
     let searched = 0

lib/par2/index.ts

@@ -0,0 +1,2 @@
+/** Par2 FileDesc packets (from voyager 1's lib/par2): the files a par2 set covers. */
+export { parsePar2, isPar2, type Par2File, type Par2Set } from './parse.js'

lib/par2/parse.ts

@@ -0,0 +1,74 @@
+import { createHash } from 'node:crypto'
+
+export interface Par2File {
+  /** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */
+  fileId: string
+  /** MD5 of the whole file (hex). */
+  md5: string
+  /** MD5 of the first 16 KiB (hex). */
+  md5_16k: string
+  length: number
+  name: string
+}
+
+export interface Par2Set {
+  /** Recovery set ID (hex), shared by every packet of one par2 set. */
+  recoverySetId: string
+  files: Par2File[]
+}
+
+const MAGIC = Buffer.from('PAR2\0PKT', 'latin1')
+const HEADER_SIZE = 64
+const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1')
+
+export function isPar2(data: Buffer): boolean {
+  return data.subarray(0, MAGIC.length).equals(MAGIC)
+}
+
+/**
+ * Reads the FileDesc packets of a par2 file (index or recovery volume).
+ * Packets with a bad length or checksum are skipped; parsing resumes at the
+ * next magic sequence. Returns null when no valid FileDesc packet is found.
+ */
+export function parsePar2(data: Buffer): Par2Set | null {
+  const files = new Map<string, Par2File>()
+  let recoverySetId: string | undefined
+  let offset = data.indexOf(MAGIC)
+
+  while (offset !== -1 && offset + HEADER_SIZE <= data.length) {
+    const length = Number(data.readBigUInt64LE(offset + 8))
+    const valid =
+      length >= HEADER_SIZE &&
+      length % 4 === 0 &&
+      offset + length <= data.length &&
+      createHash('md5')
+        .update(data.subarray(offset + 32, offset + length))
+        .digest()
+        .equals(data.subarray(offset + 16, offset + 32))
+
+    if (!valid) {
+      offset = data.indexOf(MAGIC, offset + 1)
+      continue
+    }
+
+    const setId = data.subarray(offset + 32, offset + 48).toString('hex')
+    const type = data.subarray(offset + 48, offset + 64)
+    if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) {
+      recoverySetId = setId
+      const body = data.subarray(offset + HEADER_SIZE, offset + length)
+      const fileId = body.subarray(0, 16).toString('hex')
+      files.set(fileId, {
+        fileId,
+        md5: body.subarray(16, 32).toString('hex'),
+        md5_16k: body.subarray(32, 48).toString('hex'),
+        length: Number(body.readBigUInt64LE(48)),
+        // The name is padded with NULs to a multiple of 4 bytes.
+        name: body.subarray(56).toString('utf8').replace(/\0+$/, ''),
+      })
+    }
+    offset += length
+  }
+
+  if (!recoverySetId || files.size === 0) return null
+  return { recoverySetId, files: [...files.values()] }
+}

lib/probe/index.ts

@@ -0,0 +1 @@
+export { inspect, innerName, MAX_NAMES, type Inspection } from './inspect.js'

lib/probe/inspect.ts

@@ -0,0 +1,79 @@
+import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
+import { isPar2, parsePar2 } from '#lib/par2/index'
+import { parseRarHeaders } from '#lib/rar/index'
+import { sniffContentType, type ContentType } from '#lib/content/index'
+
+/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
+export interface Inspection {
+  /** The file name in the yEnc header. */
+  yencName: string
+  /** The whole file's size, from the yEnc header. */
+  size: number
+  type: ContentType | null
+  /** Names inside: an archive's entries or a par2 set's files, the largest first, at most MAX_NAMES. */
+  names?: string[]
+  /** How many entries or files there are in all. */
+  entries?: number
+  /** An archive with encrypted entries or headers. */
+  passworded?: boolean
+}
+
+/** Names kept per inspection: the catalog wants the content's name, not every volume's. */
+export const MAX_NAMES = 8
+const MAX_NAME_LENGTH = 200
+
+/** Archive volumes, par2 volumes and split parts: names of the packaging, not the content. */
+const PACKAGING = /\.(rar|r\d{2,3}|s\d{2}|\d{3}|7z(\.\d+)?|zip|z\d{2}|par2|sfv|nfo|nzb)$/i
+
+/** Decodes a body; some tools declare a size that leaves out a header, the CRC still has to match. */
+function decodeBody(body: Buffer): DecodedSegment {
+  try {
+    return decodeSegment(body)
+  } catch (error) {
+    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+    return decodeSegment(body, { lenientSize: true })
+  }
+}
+
+function largestFirst(items: { name: string; size: number }[]): string[] {
+  return items
+    .sort((a, b) => b.size - a.size)
+    .slice(0, MAX_NAMES)
+    .map((i) => i.name.slice(0, MAX_NAME_LENGTH))
+}
+
+/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */
+export function inspect(body: Buffer): Inspection {
+  const segment = decodeBody(body)
+  const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
+  if (isPar2(segment.data)) {
+    // An index file fits one article; what a larger one cut off is skipped by the parser.
+    const set = parsePar2(segment.data)
+    if (set) {
+      result.entries = set.files.length
+      result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
+    }
+    return result
+  }
+  if (result.type === 'rar4' || result.type === 'rar5') {
+    const info = parseRarHeaders(segment.data)
+    if (info) {
+      const files = info.entries.filter((e) => !e.directory)
+      result.entries = files.length
+      result.names = largestFirst(files.map((e) => ({ name: e.name, size: Number(e.size ?? 0) })))
+      result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
+    }
+  }
+  return result
+}
+
+/**
+ * A release's best inner name from its inspections: the largest named entry
+ * of its archive, else the largest file of its par2 set that isn't
+ * packaging, else a lone file's yEnc name. Null when none says more.
+ */
+export function innerName(inspections: Partial<Record<string, Inspection>>): string | null {
+  const content = (i?: Inspection) => i?.names?.find((n) => !PACKAGING.test(n))
+  const pick = content(inspections.archive) ?? content(inspections.par2) ?? (inspections.lone?.yencName || undefined)
+  return pick ? pick.replace(/^.*[\\/]/, '').slice(0, 255) : null
+}

lib/rar/index.ts

@@ -0,0 +1,2 @@
+/** RAR 4 and 5 headers (from voyager 1's lib/rar): a volume's listing, encryption, volume flags. */
+export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'

lib/rar/parse.ts

@@ -0,0 +1,351 @@
+import { crc32 } from 'node:zlib'
+
+export interface RarEntry {
+  name: string
+  /** Unpacked size in bytes (null when the archiver did not record it). */
+  size: number | null
+  packedSize: number
+  directory: boolean
+  /** The file's data is password protected. */
+  encrypted: boolean
+  /** Continues from the previous volume. */
+  splitBefore: boolean
+  /** Continues in the next volume. */
+  splitAfter: boolean
+  /** Stored without compression, so the data can be copied out as is. */
+  stored: boolean
+  /** Offset of the entry's (packed) data in the parsed buffer. */
+  dataOffset: number
+}
+
+export interface RarInfo {
+  /** 7z listings take the same shape (see lib/sevenzip). */
+  format: 'rar4' | 'rar5' | '7z'
+  /** Part of a multi-volume set. */
+  volume: boolean
+  /** First volume of the set (null when the format version does not say). */
+  firstVolume: boolean | null
+  solid: boolean
+  /** File names are encrypted too; nothing can be listed without the password. */
+  headersEncrypted: boolean
+  entries: RarEntry[]
+  /** The buffer ended before the end-of-archive header (normal for one segment). */
+  truncated: boolean
+  /**
+   * Offset of the first block that was not read: where the next header
+   * starts (possibly beyond the buffer, after an entry's data), or the
+   * header the buffer cut off. After the end-of-archive header, its end.
+   */
+  next: number
+}
+
+const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
+const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
+
+export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
+  if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5'
+  if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4'
+  return null
+}
+
+/**
+ * Lists a RAR archive from its headers only, so the first segment of a first
+ * volume is enough. Returns null when the data is not a RAR archive.
+ *
+ * With `continueAs`, `data` starts at a header in the middle of a volume (no
+ * signature), e.g. where a previous parse said the next header is.
+ */
+export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
+  if (continueAs === 'rar4') return parseRar4(data, 0)
+  if (continueAs === 'rar5') return parseRar5(data, 0)
+  const format = rarFormat(data)
+  if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
+  if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
+  return null
+}
+
+// --- RAR 4.x (RAR 1.5 – 4.x archive format) -------------------------------
+
+const R4_MAIN = 0x73
+const R4_FILE = 0x74
+const R4_END = 0x7b
+const R4_LONG_BLOCK = 0x8000
+
+function parseRar4(data: Buffer, start: number): RarInfo {
+  const info: RarInfo = {
+    format: 'rar4',
+    volume: false,
+    firstVolume: null,
+    solid: false,
+    headersEncrypted: false,
+    entries: [],
+    truncated: true,
+    next: start,
+  }
+  let p = start
+
+  while (p + 7 <= data.length) {
+    const headCrc = data.readUInt16LE(p)
+    const type = data[p + 2]
+    const flags = data.readUInt16LE(p + 3)
+    const size = data.readUInt16LE(p + 5)
+    if (size < 7 || p + size > data.length) break
+
+    if (type === R4_END) {
+      info.truncated = false
+      p += size
+      break
+    }
+
+    let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0)
+
+    if (type === R4_MAIN) {
+      info.volume = Boolean(flags & 0x0001)
+      info.solid = Boolean(flags & 0x0008)
+      info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true
+      if (flags & 0x0080) {
+        info.headersEncrypted = true
+        break
+      }
+    } else if (type === R4_FILE) {
+      if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break
+      const large = Boolean(flags & 0x0100)
+      let packed = data.readUInt32LE(p + 7)
+      let unpacked = data.readUInt32LE(p + 11)
+      const nameSize = data.readUInt16LE(p + 26)
+      let nameStart = p + 32
+      if (large) {
+        packed += data.readUInt32LE(p + 32) * 2 ** 32
+        unpacked += data.readUInt32LE(p + 36) * 2 ** 32
+        nameStart = p + 40
+      }
+      const rawName = data.subarray(nameStart, nameStart + nameSize)
+      info.entries.push({
+        name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'),
+        // 0xffffffff with LHD_LARGE unset means "unknown" in some archivers.
+        size: unpacked === 0xffffffff && !large ? null : unpacked,
+        packedSize: packed,
+        directory: (flags & 0x00e0) === 0x00e0,
+        encrypted: Boolean(flags & 0x0004),
+        splitBefore: Boolean(flags & 0x0001),
+        splitAfter: Boolean(flags & 0x0002),
+        stored: data[p + 25] === 0x30,
+        dataOffset: p + size,
+      })
+      next = p + size + packed
+    }
+
+    if (next <= p) break
+    p = next
+  }
+  // Archivers before RAR 3.0 don't set the first-volume flag; a first
+  // volume is one whose first file does not continue from a previous one.
+  if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
+    info.firstVolume = true
+  }
+  info.next = p
+  return info
+}
+
+/**
+ * RAR 4 stores Unicode names as "ASCII\0<compressed UTF-16>", where the
+ * compressed part refers back to the ASCII bytes (unrar's DecodeFileName).
+ */
+export function decodeRar4Name(raw: Buffer): string {
+  const zero = raw.indexOf(0)
+  if (zero === -1) return raw.toString('utf8')
+  const ascii = raw.subarray(0, zero)
+  const enc = raw.subarray(zero + 1)
+  if (enc.length === 0) return ascii.toString('latin1')
+
+  const out: number[] = []
+  const highByte = enc[0]
+  let encPos = 1
+  let flags = 0
+  let flagBits = 0
+  while (encPos < enc.length) {
+    if (flagBits === 0) {
+      flags = enc[encPos++]
+      flagBits = 8
+    }
+    switch (flags >> 6) {
+      case 0:
+        out.push(enc[encPos++])
+        break
+      case 1:
+        out.push(enc[encPos++] + (highByte << 8))
+        break
+      case 2:
+        out.push(enc[encPos] + (enc[encPos + 1] << 8))
+        encPos += 2
+        break
+      case 3: {
+        let length = enc[encPos++]
+        if (length & 0x80) {
+          const correction = enc[encPos++]
+          for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) {
+            out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8))
+          }
+        } else {
+          for (length += 2; length > 0 && out.length < ascii.length; length--) {
+            out.push(ascii[out.length])
+          }
+        }
+        break
+      }
+    }
+    flags = (flags << 2) & 0xff
+    flagBits -= 2
+  }
+  return String.fromCharCode(...out)
+}
+
+// --- RAR 5.0 ----------------------------------------------------------------
+
+const R5_MAIN = 1
+const R5_FILE = 2
+const R5_ENCRYPTION = 4
+const R5_END = 5
+
+/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */
+function vint(data: Buffer, offset: number): { value: number; next: number } | null {
+  let value = 0
+  let shift = 1
+  for (let i = offset; i < data.length && i < offset + 10; i++) {
+    value += (data[i] & 0x7f) * shift
+    shift *= 128
+    if (!(data[i] & 0x80)) return { value, next: i + 1 }
+  }
+  return null
+}
+
+function parseRar5(data: Buffer, start: number): RarInfo {
+  const info: RarInfo = {
+    format: 'rar5',
+    volume: false,
+    firstVolume: null,
+    solid: false,
+    headersEncrypted: false,
+    entries: [],
+    truncated: true,
+    next: start,
+  }
+  let p = start
+
+  while (p + 5 <= data.length) {
+    const headCrc = data.readUInt32LE(p)
+    const headerSize = vint(data, p + 4)
+    if (!headerSize || headerSize.value === 0) break
+    const headerEnd = headerSize.next + headerSize.value
+    if (headerEnd > data.length) break
+    if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break
+
+    const type = vint(data, headerSize.next)
+    const flags = type && vint(data, type.next)
+    if (!type || !flags) break
+    let cursor = flags.next
+    let extraSize = 0
+    let dataSize = 0
+    if (flags.value & 0x0001) {
+      const v = vint(data, cursor)
+      if (!v) break
+      extraSize = v.value
+      cursor = v.next
+    }
+    if (flags.value & 0x0002) {
+      const v = vint(data, cursor)
+      if (!v) break
+      dataSize = v.value
+      cursor = v.next
+    }
+
+    if (type.value === R5_END) {
+      info.truncated = false
+      p = headerEnd
+      break
+    }
+    if (type.value === R5_ENCRYPTION) {
+      info.headersEncrypted = true
+      break
+    }
+
+    if (type.value === R5_MAIN) {
+      const archiveFlags = vint(data, cursor)
+      if (!archiveFlags) break
+      info.volume = Boolean(archiveFlags.value & 0x0001)
+      // A volume number is only stored for volumes after the first.
+      info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true
+      info.solid = Boolean(archiveFlags.value & 0x0004)
+    } else if (type.value === R5_FILE) {
+      const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize)
+      if (!entry) break
+      info.entries.push(entry)
+    }
+
+    const next = headerEnd + dataSize
+    if (next <= p) break
+    p = next
+  }
+  info.next = p
+  return info
+}
+
+function parseRar5File(
+  data: Buffer,
+  start: number,
+  extraStart: number,
+  headerEnd: number,
+  headerFlags: number,
+  dataSize: number
+): RarEntry | null {
+  const fileFlags = vint(data, start)
+  const unpacked = fileFlags && vint(data, fileFlags.next)
+  const attributes = unpacked && vint(data, unpacked.next)
+  if (!fileFlags || !unpacked || !attributes) return null
+  let cursor = attributes.next
+  if (fileFlags.value & 0x0002) cursor += 4 // mtime
+  if (fileFlags.value & 0x0004) cursor += 4 // data CRC32
+  const compression = vint(data, cursor)
+  const hostOs = compression && vint(data, compression.next)
+  const nameLength = hostOs && vint(data, hostOs.next)
+  if (!compression || !hostOs || !nameLength) return null
+  const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8')
+  // Compression information: bits 7-9 are the method, 0 is "store".
+  const method = Math.floor(compression.value / 128) & 0x7
+
+  // Extra area records: size, type, data. Type 1 is file encryption.
+  let encrypted = false
+  let e = extraStart
+  while (e < headerEnd) {
+    const recordSize = vint(data, e)
+    const recordType = recordSize && vint(data, recordSize.next)
+    if (!recordSize || !recordType || recordSize.value === 0) break
+    if (recordType.value === 0x01) encrypted = true
+    e = recordSize.next + recordSize.value
+  }
+
+  return {
+    name,
+    size: fileFlags.value & 0x0008 ? null : unpacked.value,
+    packedSize: dataSize,
+    directory: Boolean(fileFlags.value & 0x0001),
+    encrypted,
+    splitBefore: Boolean(headerFlags & 0x0008),
+    splitAfter: Boolean(headerFlags & 0x0010),
+    stored: method === 0,
+    dataOffset: headerEnd,
+  }
+}
+
+/**
+ * The data of an entry stored without compression, copied out of the
+ * archive buffer. Null when it is compressed, encrypted, split over volumes
+ * or not completely in the buffer.
+ */
+export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null {
+  if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
+  if (entry.size === null || entry.size !== entry.packedSize) return null
+  const end = entry.dataOffset + entry.packedSize
+  if (end > data.length) return null
+  return data.subarray(entry.dataOffset, end)
+}

lib/yenc/decode.ts

@@ -0,0 +1,76 @@
+import yencode from 'yencode'
+import { YencCrcError, YencError } from './errors.js'
+import { parseYencMeta, type YencMeta } from './meta.js'
+
+export interface DecodedSegment {
+  meta: YencMeta
+  /** Decoded bytes of this segment. */
+  data: Buffer
+  /** 0-based offset of `data` within the complete file. */
+  offset: number
+}
+
+export interface DecodeOptions {
+  /**
+   * Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
+   * bodies, so this defaults to true.
+   */
+  stripDots?: boolean
+  /** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
+  verifyCrc?: boolean
+  /**
+   * Accept a decoded length that differs from the declared size (the CRC is
+   * still checked). Some posting tools declare a size that leaves out a
+   * header, e.g. the 20-byte header of `nzbe` containers.
+   */
+  lenientSize?: boolean
+}
+
+export function crc32Hex(data: Buffer): string {
+  return yencode.crc32(data).toString('hex')
+}
+
+/**
+ * Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
+ * without the data itself.
+ */
+export function combineCrc32(parts: { crc: string; length: number }[]): string {
+  let combined: Buffer | undefined
+  for (const part of parts) {
+    const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
+    combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
+  }
+  return (combined ?? Buffer.alloc(4)).toString('hex')
+}
+
+/**
+ * Decodes one yEnc article body and checks its size and CRC32 against the
+ * `=yend` trailer.
+ */
+export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
+  const { stripDots = true, verifyCrc = true, lenientSize = false } = options
+  const meta = parseYencMeta(body)
+  const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
+
+  if (lenientSize) {
+    // Only the CRC below decides.
+  } else if (data.length !== meta.end.size) {
+    throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
+  }
+  if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
+    throw new YencError(
+      `Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
+    )
+  }
+
+  if (verifyCrc) {
+    // Multi-part posts carry pcrc32 per part; single-part posts only crc32.
+    const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
+    if (expected) {
+      const actual = crc32Hex(data)
+      if (actual !== expected) throw new YencCrcError(expected, actual)
+    }
+  }
+
+  return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
+}

lib/yenc/errors.ts

@@ -0,0 +1,16 @@
+export class YencError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'YencError'
+  }
+}
+
+export class YencCrcError extends YencError {
+  constructor(
+    public readonly expected: string,
+    public readonly actual: string
+  ) {
+    super(`CRC32 mismatch: expected ${expected}, got ${actual}`)
+    this.name = 'YencCrcError'
+  }
+}

lib/yenc/index.ts

@@ -0,0 +1,4 @@
+/** yEnc article bodies (from voyager 1's lib/yenc): header and trailer, decoding with size and CRC checks. */
+export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js'
+export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js'
+export { YencError, YencCrcError } from './errors.js'

lib/yenc/meta.ts

@@ -0,0 +1,140 @@
+import { YencError } from './errors.js'
+
+/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */
+export interface YencBegin {
+  line: number
+  size: number
+  name: string
+  part?: number
+  total?: number
+}
+
+/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */
+export interface YencPart {
+  begin: number
+  end: number
+}
+
+/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */
+export interface YencEnd {
+  size: number
+  part?: number
+  pcrc32?: string
+  crc32?: string
+}
+
+export interface YencMeta {
+  begin: YencBegin
+  part?: YencPart
+  end: YencEnd
+  /** Byte range of the encoded data inside the article body. */
+  dataStart: number
+  dataEnd: number
+}
+
+const CRLF = Buffer.from('\r\n')
+const BEGIN = Buffer.from('=ybegin ')
+const PART = Buffer.from('=ypart ')
+const END = Buffer.from('\r\n=yend ')
+
+/**
+ * Parses `key=value` pairs from a yEnc header line. `name` is always the last
+ * field and runs to the end of the line, because filenames may contain spaces
+ * and `=`.
+ */
+function parseFields(line: string): Record<string, string> {
+  const fields: Record<string, string> = {}
+  const nameIndex = line.search(/(^|\s)name=/)
+  let rest = line
+  if (nameIndex !== -1) {
+    const nameStart = line.indexOf('name=', nameIndex) + 'name='.length
+    fields.name = line.slice(nameStart).trim()
+    rest = line.slice(0, nameIndex)
+  }
+  for (const match of rest.matchAll(/(\w+)=(\S+)/g)) {
+    fields[match[1]] = match[2]
+  }
+  return fields
+}
+
+function toInt(fields: Record<string, string>, key: string, required: true): number
+function toInt(fields: Record<string, string>, key: string, required?: false): number | undefined
+function toInt(fields: Record<string, string>, key: string, required = false) {
+  const raw = fields[key]
+  if (raw === undefined) {
+    if (required) throw new YencError(`Missing "${key}" in yEnc header`)
+    return undefined
+  }
+  const value = Number.parseInt(raw, 10)
+  if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`)
+  return value
+}
+
+function toCrc(fields: Record<string, string>, key: string) {
+  const raw = fields[key]
+  if (raw === undefined) return undefined
+  if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`)
+  return raw.toLowerCase().padStart(8, '0')
+}
+
+/**
+ * Number of parts in the file. Many posters leave `total=` out of `=ybegin`;
+ * then it follows from the part size, since all parts except the last one
+ * are the same size.
+ */
+export function inferTotalParts(meta: Pick<YencMeta, 'begin' | 'part'>): number {
+  const { begin, part } = meta
+  if (begin.total) return begin.total
+  if (!part || begin.part === undefined) return 1
+  if (part.end >= begin.size) return begin.part
+  return Math.ceil(begin.size / (part.end - part.begin + 1))
+}
+
+function lineAt(buffer: Buffer, start: number) {
+  const eol = buffer.indexOf(CRLF, start)
+  const end = eol === -1 ? buffer.length : eol
+  return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length }
+}
+
+/**
+ * Locates and parses the yEnc header/trailer lines of one article body,
+ * without decoding the data itself.
+ */
+export function parseYencMeta(body: Buffer): YencMeta {
+  const beginIndex = body.indexOf(BEGIN)
+  if (beginIndex === -1) throw new YencError('No =ybegin line found')
+
+  const beginLine = lineAt(body, beginIndex)
+  const beginFields = parseFields(beginLine.text)
+  const begin: YencBegin = {
+    line: toInt(beginFields, 'line', true),
+    size: toInt(beginFields, 'size', true),
+    name: beginFields.name ?? '',
+    part: toInt(beginFields, 'part'),
+    total: toInt(beginFields, 'total'),
+  }
+  if (!begin.name) throw new YencError('Missing "name" in =ybegin line')
+
+  let dataStart = beginLine.next
+  let part: YencPart | undefined
+  if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) {
+    const partLine = lineAt(body, dataStart)
+    const partFields = parseFields(partLine.text)
+    part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) }
+    dataStart = partLine.next
+  }
+
+  const endIndex = body.lastIndexOf(END)
+  if (endIndex === -1 || endIndex < dataStart - CRLF.length) {
+    throw new YencError('No =yend line found')
+  }
+  const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text)
+  const end: YencEnd = {
+    size: toInt(endFields, 'size', true),
+    part: toInt(endFields, 'part'),
+    pcrc32: toCrc(endFields, 'pcrc32'),
+    crc32: toCrc(endFields, 'crc32'),
+  }
+
+  return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) }
+}

package-lock.json

@@ -22,7 +22,8 @@
         "luxon": "^3.7.2",
         "pg": "^8.23.1",
         "reflect-metadata": "^0.2.2",
-        "xmlbuilder2": "^4.0.3"
+        "xmlbuilder2": "^4.0.3",
+        "yencode": "^1.2.4"
       },
       "devDependencies": {
         "@adonisjs/assembler": "^8.5.2",
@@ -7364,6 +7365,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=23"
       }
     },
+    "node_modules/yencode": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/yencode/-/yencode-1.2.4.tgz",
+      "integrity": "sha512-AfQkwlxmgfoWq04ATRUaA6YSjpclGzHNRt2BP76DogqvuiNUtuIj1Fq/fAThQmOwEXWlbQ6sMc/qjdeFtyCScQ==",
+      "hasInstallScript": true,
+      "license": "CC0-1.0",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

package.json

@@ -70,7 +70,8 @@
     "luxon": "^3.7.2",
     "pg": "^8.23.1",
     "reflect-metadata": "^0.2.2",
-    "xmlbuilder2": "^4.0.3"
+    "xmlbuilder2": "^4.0.3",
+    "yencode": "^1.2.4"
   },
   "hotHook": {
     "boundaries": [

start/env.ts

@@ -64,5 +64,6 @@ export default await Env.create(new URL('../', import.meta.url), {
   SPOOL_MIN_AHEAD: Env.schema.number.optional(),
   SPOOL_MAX_GB: Env.schema.number.optional(),
   PREFETCH_PER_RANGE: Env.schema.number.optional(),
+  PROBE_CONNECTIONS: Env.schema.number.optional(),
   PREFETCH_PER_GROUP: Env.schema.number.optional(),
 })

tests/unit/nntp/xover_client.spec.ts

@@ -1,7 +1,7 @@
 import { test } from '@japa/runner'
 import net from 'node:net'
 import { deflateSync } from 'node:zlib'
-import { XoverClient, parseOverviews } from '#lib/nntp/index'
+import { MissingArticle, XoverClient, parseOverviews } from '#lib/nntp/index'
 import { unzipSync } from 'node:zlib'
 
 const LINES = [
@@ -49,6 +49,11 @@ function server(): Promise<{ port: number; close: () => void }> {
               sock.write('215 NewsGroups Follow\r\n')
               send(Buffer.concat([body, Buffer.from('.\r\n')]))
             }
+          } else if (line === 'BODY <gone@test>') sock.write('430 No Such Article\r\n')
+          else if (line.startsWith('BODY')) {
+            sock.write('222 0 <one@test> body\r\n')
+            // Raw bytes, a dot-stuffed line among them.
+            send(Buffer.concat([Buffer.from([0x3d, 0x79, 0xff, 0x00, 0x0d, 0x0a]), Buffer.from('..stuffed\r\nlast\r\n.\r\n')]))
           } else if (line === 'QUIT') sock.end('205 bye\r\n')
         }
       })
@@ -58,6 +63,22 @@ function server(): Promise<{ port: number; close: () => void }> {
 }
 
 test.group('nntp / XoverClient', () => {
+  test('reads a body by message-id as bytes, raw; 430 is MissingArticle', async ({ assert }) => {
+    const { port, close } = await server()
+    const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress: false })
+    try {
+      const body = await client.body('one@test')
+      assert.deepEqual([...body.subarray(0, 6)], [0x3d, 0x79, 0xff, 0x00, 0x0d, 0x0a])
+      assert.equal(body.subarray(6).toString('latin1'), '..stuffed\r\nlast\r\n')
+      await assert.rejects(() => client.body('<gone@test>'), MissingArticle)
+      // The connection is still usable after both.
+      assert.equal((await client.body('<one@test>')).length, body.length)
+    } finally {
+      client.close()
+      close()
+    }
+  })
+
   test('parses overview lines into the fields nntp-js gives', ({ assert }) => {
     const [[number, fields]] = parseOverviews(LINES[0] + '\r\n')
     assert.equal(number, 101)

tests/unit/probe/inspect.spec.ts

@@ -0,0 +1,124 @@
+import { test } from '@japa/runner'
+import { createHash } from 'node:crypto'
+import { crc32 } from 'node:zlib'
+import yencode from 'yencode'
+import { crc32Hex } from '#lib/yenc/index'
+import { inspect, innerName, MAX_NAMES } from '#lib/probe/index'
+
+/** A single-part yEnc article of `data`. */
+function article(data: Buffer, name: string) {
+  const lines = [`=ybegin line=128 size=${data.length} name=${name}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} crc32=${crc32Hex(data)}`]
+  return Buffer.from(lines.join('\r\n'), 'latin1')
+}
+
+const md5 = (data: Buffer) => createHash('md5').update(data).digest()
+/** Builds one PAR2 packet with a valid checksum. */
+function packet(setId: Buffer, type: string, body: Buffer) {
+  const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
+  const length = Buffer.alloc(8)
+  length.writeBigUInt64LE(BigInt(64 + padded.length))
+  const typeBuf = Buffer.alloc(16)
+  typeBuf.write(type, 'latin1')
+  const hashed = Buffer.concat([setId, typeBuf, padded])
+  return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
+}
+function fileDesc(setId: Buffer, name: string, size: number) {
+  const len = Buffer.alloc(8)
+  len.writeBigUInt64LE(BigInt(size))
+  const md5_16k = md5(Buffer.from(`${name}-16k`))
+  const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
+  const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
+  return packet(setId, 'PAR 2.0\0FileDesc', body)
+}
+const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
+function vint(value: number) {
+  const bytes: number[] = []
+  do {
+    let byte = value % 128
+    value = Math.floor(value / 128)
+    if (value > 0) byte |= 0x80
+    bytes.push(byte)
+  } while (value > 0)
+  return Buffer.from(bytes)
+}
+function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
+  let flags = headerFlags
+  const parts = [vint(type)]
+  const sizes: Buffer[] = []
+  if (opts.extra) {
+    flags |= 0x0001
+    sizes.push(vint(opts.extra.length))
+  }
+  if (opts.data !== undefined) {
+    flags |= 0x0002
+    sizes.push(vint(opts.data))
+  }
+  const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
+  const sized = Buffer.concat([vint(header.length), header])
+  const crc = Buffer.alloc(4)
+  crc.writeUInt32LE(crc32(sized) >>> 0)
+  return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
+}
+function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
+  const nameBuf = Buffer.from(name, 'utf8')
+  const fields = Buffer.concat([
+    vint(opts.dir ? 0x0001 : 0), // file flags
+    vint(size),
+    vint(0x20), // attributes
+    vint(0), // compression info
+    vint(0), // host OS
+    vint(nameBuf.length),
+    nameBuf,
+  ])
+  // Encryption record: size, type 1, then its data (dummy here).
+  const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
+  return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
+}
+
+test.group('probe / inspect', () => {
+  test('a par2 index: its files, largest first', ({ assert }) => {
+    const setId = Buffer.alloc(16, 3)
+    const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)])
+    const result = inspect(article(par2, 'a8f3k2.par2'))
+    assert.deepEqual(result, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
+  })
+
+  test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => {
+    const rar = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('Movie.2020.1080p/movie.mkv', 5_000_000, { packed: 10 }), rar5File('Movie.2020.1080p/sample.mkv', 1000, { packed: 10 })])
+    const open = inspect(article(rar, 'x7q.part01.rar'))
+    assert.include(open, { type: 'rar5', entries: 2, passworded: false })
+    assert.deepEqual(open.names, ['Movie.2020.1080p/movie.mkv', 'Movie.2020.1080p/sample.mkv'])
+    const locked = inspect(article(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]), 'y.rar'))
+    assert.isTrue(locked.passworded)
+    const hidden = inspect(article(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]), 'z.rar'))
+    assert.include(hidden, { passworded: true, entries: 0 })
+  })
+
+  test('other data: the yEnc name and what the bytes are', ({ assert }) => {
+    const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)])
+    assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska' })
+  })
+
+  test('not yEnc: throws', ({ assert }) => {
+    assert.throws(() => inspect(Buffer.from('just some text\r\n')))
+  })
+
+  test('at most MAX_NAMES names', ({ assert }) => {
+    const setId = Buffer.alloc(16, 4)
+    const par2 = Buffer.concat(Array.from({ length: MAX_NAMES + 5 }, (_, i) => fileDesc(setId, `f${i}.rar`, i)))
+    const result = inspect(article(par2, 'p.par2'))
+    assert.equal(result.entries, MAX_NAMES + 5)
+    assert.lengthOf(result.names!, MAX_NAMES)
+  })
+})
+
+test.group('probe / innerName', () => {
+  test("the archive's largest entry, else par2's largest non-packaging file, else a lone file's yEnc name", ({ assert }) => {
+    const archive = { yencName: 'a.rar', size: 1, type: 'rar5' as const, names: ['Show.S01E01/show.s01e01.mkv'] }
+    const par2 = { yencName: 'a.par2', size: 1, type: 'par2' as const, names: ['abc.part01.rar', 'Show.S01E01.1080p.mkv'] }
+    assert.equal(innerName({ archive, par2 }), 'show.s01e01.mkv')
+    assert.equal(innerName({ par2 }), 'Show.S01E01.1080p.mkv')
+    assert.equal(innerName({ par2: { ...par2, names: ['abc.part01.rar', 'abc.par2'] } }), null)
+    assert.equal(innerName({ lone: { yencName: 'Real.Name.mkv', size: 1, type: 'matroska' } }), 'Real.Name.mkv')
+  })
+})

tests/unit/probe/par2.spec.ts

@@ -0,0 +1,60 @@
+import { test } from '@japa/runner'
+import { createHash } from 'node:crypto'
+import { isPar2, parsePar2 } from '#lib/par2/index'
+
+const md5 = (data: Buffer) => createHash('md5').update(data).digest()
+
+/** Builds one PAR2 packet with a valid checksum. */
+function packet(setId: Buffer, type: string, body: Buffer) {
+  const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
+  const length = Buffer.alloc(8)
+  length.writeBigUInt64LE(BigInt(64 + padded.length))
+  const typeBuf = Buffer.alloc(16)
+  typeBuf.write(type, 'latin1')
+  const hashed = Buffer.concat([setId, typeBuf, padded])
+  return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
+}
+
+function fileDesc(setId: Buffer, name: string, size: number) {
+  const len = Buffer.alloc(8)
+  len.writeBigUInt64LE(BigInt(size))
+  const md5_16k = md5(Buffer.from(`${name}-16k`))
+  const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
+  const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
+  return packet(setId, 'PAR 2.0\0FileDesc', body)
+}
+
+test.group('par2 / parsePar2', () => {
+  const setId = Buffer.alloc(16, 7)
+
+  test('reads file names and sizes from FileDesc packets', ({ assert }) => {
+    const data = Buffer.concat([
+      packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)),
+      fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600),
+      fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800),
+      fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common
+    ])
+    assert.isTrue(isPar2(data))
+    const set = parsePar2(data)!
+    assert.equal(set.recoverySetId, setId.toString('hex'))
+    assert.deepEqual(
+      set.files.map((f) => [f.name, f.length]),
+      [
+        ['Some.Movie.2024.part01.rar', 104857600],
+        ['Some.Movie.2024.part02.rar', 52428800],
+      ]
+    )
+  })
+
+  test('skips corrupted packets and resumes at the next one', ({ assert }) => {
+    const bad = fileDesc(setId, 'bad.bin', 1)
+    bad[100] ^= 0xff
+    const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)])
+    assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin'])
+  })
+
+  test('returns null for non-par2 data', ({ assert }) => {
+    assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1')))
+    assert.isNull(parsePar2(Buffer.from('hello world')))
+  })
+})

tests/unit/probe/rar.spec.ts

@@ -0,0 +1,189 @@
+import { test } from '@japa/runner'
+import { crc32 } from 'node:zlib'
+import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, type RarEntry } from '#lib/rar/index'
+
+/** Entries without the extraction fields, for comparing listings. */
+const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
+
+// --- RAR 4 builders -----------------------------------------------------------
+
+const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
+
+function rar4Block(type: number, flags: number, body: Buffer) {
+  const header = Buffer.alloc(7)
+  header.writeUInt8(type, 2)
+  header.writeUInt16LE(flags, 3)
+  header.writeUInt16LE(7 + body.length, 5)
+  const block = Buffer.concat([header, body])
+  block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0)
+  return block
+}
+
+function rar4Main(flags: number) {
+  return rar4Block(0x73, flags, Buffer.alloc(6))
+}
+
+function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) {
+  const body = Buffer.alloc(25)
+  body.writeUInt8(opts.method ?? 0x33, 18)
+  body.writeUInt32LE(opts.packed, 0)
+  body.writeUInt32LE(opts.unpacked, 4)
+  body.writeUInt16LE(name.length, 19)
+  // LONG_BLOCK (0x8000) is always set on file headers.
+  return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)])
+}
+
+// --- RAR 5 builders -----------------------------------------------------------
+
+const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
+
+function vint(value: number) {
+  const bytes: number[] = []
+  do {
+    let byte = value % 128
+    value = Math.floor(value / 128)
+    if (value > 0) byte |= 0x80
+    bytes.push(byte)
+  } while (value > 0)
+  return Buffer.from(bytes)
+}
+
+function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
+  let flags = headerFlags
+  const parts = [vint(type)]
+  const sizes: Buffer[] = []
+  if (opts.extra) {
+    flags |= 0x0001
+    sizes.push(vint(opts.extra.length))
+  }
+  if (opts.data !== undefined) {
+    flags |= 0x0002
+    sizes.push(vint(opts.data))
+  }
+  const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
+  const sized = Buffer.concat([vint(header.length), header])
+  const crc = Buffer.alloc(4)
+  crc.writeUInt32LE(crc32(sized) >>> 0)
+  return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
+}
+
+function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
+  const nameBuf = Buffer.from(name, 'utf8')
+  const fields = Buffer.concat([
+    vint(opts.dir ? 0x0001 : 0), // file flags
+    vint(size),
+    vint(0x20), // attributes
+    vint(0), // compression info
+    vint(0), // host OS
+    vint(nameBuf.length),
+    nameBuf,
+  ])
+  // Encryption record: size, type 1, then its data (dummy here).
+  const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
+  return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
+}
+
+test.group('rar / RAR 4', () => {
+  test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => {
+    const data = Buffer.concat([
+      RAR4,
+      rar4Main(0x0001 | 0x0100), // volume, first volume
+      rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }),
+    ])
+    assert.equal(rarFormat(data), 'rar4')
+    const info = parseRarHeaders(data)!
+    assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true })
+    assert.deepEqual(listing(info.entries), [
+      { name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
+    ])
+  })
+
+  test('detects password-protected files and encrypted headers', ({ assert }) => {
+    const files = parseRarHeaders(
+      Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })])
+    )!
+    assert.isTrue(files.entries[0].encrypted)
+    const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))!
+    assert.isTrue(headers.headersEncrypted)
+    assert.lengthOf(headers.entries, 0)
+  })
+
+  test('copies out stored entries, not compressed ones', ({ assert }) => {
+    const data = Buffer.concat([
+      RAR4,
+      rar4Main(0),
+      rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }),
+      rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }),
+    ])
+    const [stored, packed] = parseRarHeaders(data)!.entries
+    data.write('<nzb>', stored.dataOffset)
+    assert.equal(storedEntryData(data, stored)?.toString(), '<nzb>')
+    assert.isFalse(packed.stored)
+    assert.isNull(storedEntryData(data, packed))
+  })
+
+  test('stops at a header with a bad CRC', ({ assert }) => {
+    const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 })
+    const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 })
+    bad[10] ^= 0xff
+    const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))!
+    assert.deepEqual(info.entries.map((e) => e.name), ['a.bin'])
+  })
+
+  test('decodes compressed Unicode names', ({ assert }) => {
+    // "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte,
+    // then 3 plain bytes), followed by one more flag byte for the last byte.
+    const ascii = Buffer.from('?.txt', 'latin1')
+    const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74])
+    assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt')
+    assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt')
+  })
+})
+
+test.group('rar / RAR 5', () => {
+  test('lists files and reads volume flags', ({ assert }) => {
+    const data = Buffer.concat([
+      RAR5,
+      rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume
+      rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }),
+    ])
+    assert.equal(rarFormat(data), 'rar5')
+    const info = parseRarHeaders(data)!
+    assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false })
+    assert.deepEqual(listing(info.entries), [
+      { name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
+    ])
+  })
+
+  test('a later volume carries a volume number', ({ assert }) => {
+    const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))!
+    assert.isFalse(info.firstVolume)
+  })
+
+  test('detects encrypted files and encrypted headers', ({ assert }) => {
+    const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))!
+    assert.isTrue(files.entries[0].encrypted)
+    const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))!
+    assert.isTrue(headers.headersEncrypted)
+  })
+
+  test('marks the listing complete at the end-of-archive header', ({ assert }) => {
+    const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))!
+    assert.isFalse(info.truncated)
+    assert.lengthOf(info.entries, 1)
+  })
+
+  test('copies out entries stored without compression', ({ assert }) => {
+    const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))])
+    const entry = parseRarHeaders(data)!.entries[0]
+    assert.isTrue(entry.stored)
+    data.write('<nzb>', entry.dataOffset)
+    assert.equal(storedEntryData(data, entry)?.toString(), '<nzb>')
+    assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry))
+    assert.isNull(storedEntryData(data, { ...entry, encrypted: true }))
+  })
+
+  test('returns null for other data', ({ assert }) => {
+    assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1')))
+  })
+})

tests/unit/probe/yenc.spec.ts

@@ -0,0 +1,107 @@
+import { test } from '@japa/runner'
+import yencode from 'yencode'
+import {
+  YencCrcError,
+  YencError,
+  crc32Hex,
+  decodeSegment,
+  parseYencMeta,
+} from '#lib/yenc/index'
+
+/** Builds an article body as a yEnc poster would send it. */
+function article(
+  data: Buffer,
+  opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string }
+) {
+  const fileSize = opts.fileSize ?? data.length
+  const encoded = yencode.encode(data).toString('latin1')
+  const lines: string[] = []
+  if (opts.part) {
+    lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`)
+    const begin = opts.begin ?? 1
+    lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`)
+    lines.push(encoded)
+    const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : ''
+    lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`)
+  } else {
+    lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`)
+    lines.push(encoded)
+    lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`)
+  }
+  return Buffer.from(lines.join('\r\n'), 'latin1')
+}
+
+function randomBytes(length: number, seed = 1) {
+  const buf = Buffer.alloc(length)
+  let x = seed
+  for (let i = 0; i < length; i++) {
+    x = (x * 1103515245 + 12345) & 0x7fffffff
+    buf[i] = x & 0xff
+  }
+  return buf
+}
+
+test.group('yenc / parseYencMeta', () => {
+  test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => {
+    const body = Buffer.from(
+      '=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' +
+        '=ypart begin=21120001 end=21504000\r\n' +
+        'xyz\r\n' +
+        '=yend size=384000 part=56 pcrc32=B6339AFC',
+      'latin1'
+    )
+    const meta = parseYencMeta(body)
+    assert.deepEqual(meta.begin, {
+      part: 56,
+      total: undefined,
+      line: 128,
+      size: 104857600,
+      name: 'obrMkrDeLmbhaouwYO34.part08.rar',
+    })
+    assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 })
+    assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined })
+    assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz')
+  })
+
+  test('keeps spaces and "=" in the filename', ({ assert }) => {
+    const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' })
+    assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo')
+  })
+
+  test('ignores leading blank lines before =ybegin', ({ assert }) => {
+    const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })])
+    assert.equal(parseYencMeta(body).begin.name, 'a.txt')
+  })
+
+  test('rejects bodies without yEnc markers', ({ assert }) => {
+    assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/)
+    assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/)
+  })
+})
+
+test.group('yenc / decodeSegment', () => {
+  test('round-trips a single-part post and checks crc32', ({ assert }) => {
+    const data = randomBytes(5000)
+    const segment = decodeSegment(article(data, { name: 'single.bin' }))
+    assert.isTrue(segment.data.equals(data))
+    assert.equal(segment.offset, 0)
+  })
+
+  test('throws YencCrcError on a corrupted part', ({ assert }) => {
+    const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 })
+    const index = body.indexOf('\r\n=ypart') + 60
+    body[index] = body[index] === 0x41 ? 0x42 : 0x41
+    assert.throws(() => decodeSegment(body), YencCrcError as any)
+  })
+
+  test('undoes NNTP dot-stuffing', ({ assert }) => {
+    // Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..".
+    const data = Buffer.from([4, 5, 6])
+    const body = Buffer.from(
+      `=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`,
+      'latin1'
+    )
+    assert.isTrue(decodeSegment(body).data.equals(data))
+    assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any)
+  })
+})

types/yencode.d.ts

@@ -0,0 +1,15 @@
+declare module 'yencode' {
+  export function decode(data: Buffer, stripDots?: boolean): Buffer
+  export function encode(data: Buffer, lineSize?: number, column?: number): Buffer
+  /** Returns the CRC32 as a 4-byte big-endian buffer. */
+  export function crc32(data: Buffer, initial?: Buffer): Buffer
+  /** CRC32 of A+B from crc(A), crc(B) and the length of B. */
+  export function crc32_combine(crc1: Buffer, crc2: Buffer, len2: number): Buffer
+  const yencode: {
+    decode: typeof decode
+    encode: typeof encode
+    crc32: typeof crc32
+    crc32_combine: typeof crc32_combine
+  }
+  export default yencode
+}