Rebuild: phase 2's prober reads the probe queue

probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims
waiting candidates of settled releases (lone, then par2, then archive),
fetches each article with BODY over connections of its own (no
compression) and inspects it: the yEnc name and size, the content type
from the first bytes, a par2 set's files, a RAR volume's entries and
whether they're encrypted. Results per release and kind in
release_inspections (CHUNK_TABLESPACE, the HDD); on the release
passworded, inner_name (the largest content name) and inspected_at.
430 = missing, not yEnc = error, both final; connection trouble puts
the candidate back.

Fetch and parse in one process: an article takes ~0.1 s per connection,
parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc,
lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body;
yencode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/yenc/decode.ts

@@ -0,0 +1,76 @@
+import yencode from 'yencode'
+import { YencCrcError, YencError } from './errors.js'
+import { parseYencMeta, type YencMeta } from './meta.js'
+
+export interface DecodedSegment {
+  meta: YencMeta
+  /** Decoded bytes of this segment. */
+  data: Buffer
+  /** 0-based offset of `data` within the complete file. */
+  offset: number
+}
+
+export interface DecodeOptions {
+  /**
+   * Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
+   * bodies, so this defaults to true.
+   */
+  stripDots?: boolean
+  /** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
+  verifyCrc?: boolean
+  /**
+   * Accept a decoded length that differs from the declared size (the CRC is
+   * still checked). Some posting tools declare a size that leaves out a
+   * header, e.g. the 20-byte header of `nzbe` containers.
+   */
+  lenientSize?: boolean
+}
+
+export function crc32Hex(data: Buffer): string {
+  return yencode.crc32(data).toString('hex')
+}
+
+/**
+ * Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
+ * without the data itself.
+ */
+export function combineCrc32(parts: { crc: string; length: number }[]): string {
+  let combined: Buffer | undefined
+  for (const part of parts) {
+    const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
+    combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
+  }
+  return (combined ?? Buffer.alloc(4)).toString('hex')
+}
+
+/**
+ * Decodes one yEnc article body and checks its size and CRC32 against the
+ * `=yend` trailer.
+ */
+export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
+  const { stripDots = true, verifyCrc = true, lenientSize = false } = options
+  const meta = parseYencMeta(body)
+  const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
+
+  if (lenientSize) {
+    // Only the CRC below decides.
+  } else if (data.length !== meta.end.size) {
+    throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
+  }
+  if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
+    throw new YencError(
+      `Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
+    )
+  }
+
+  if (verifyCrc) {
+    // Multi-part posts carry pcrc32 per part; single-part posts only crc32.
+    const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
+    if (expected) {
+      const actual = crc32Hex(data)
+      if (actual !== expected) throw new YencCrcError(expected, actual)
+    }
+  }
+
+  return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
+}

lib/yenc/errors.ts

@@ -0,0 +1,16 @@
+export class YencError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'YencError'
+  }
+}
+
+export class YencCrcError extends YencError {
+  constructor(
+    public readonly expected: string,
+    public readonly actual: string
+  ) {
+    super(`CRC32 mismatch: expected ${expected}, got ${actual}`)
+    this.name = 'YencCrcError'
+  }
+}

lib/yenc/index.ts

@@ -0,0 +1,4 @@
+/** yEnc article bodies (from voyager 1's lib/yenc): header and trailer, decoding with size and CRC checks. */
+export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js'
+export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js'
+export { YencError, YencCrcError } from './errors.js'

lib/yenc/meta.ts

@@ -0,0 +1,140 @@
+import { YencError } from './errors.js'
+
+/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */
+export interface YencBegin {
+  line: number
+  size: number
+  name: string
+  part?: number
+  total?: number
+}
+
+/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */
+export interface YencPart {
+  begin: number
+  end: number
+}
+
+/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */
+export interface YencEnd {
+  size: number
+  part?: number
+  pcrc32?: string
+  crc32?: string
+}
+
+export interface YencMeta {
+  begin: YencBegin
+  part?: YencPart
+  end: YencEnd
+  /** Byte range of the encoded data inside the article body. */
+  dataStart: number
+  dataEnd: number
+}
+
+const CRLF = Buffer.from('\r\n')
+const BEGIN = Buffer.from('=ybegin ')
+const PART = Buffer.from('=ypart ')
+const END = Buffer.from('\r\n=yend ')
+
+/**
+ * Parses `key=value` pairs from a yEnc header line. `name` is always the last
+ * field and runs to the end of the line, because filenames may contain spaces
+ * and `=`.
+ */
+function parseFields(line: string): Record<string, string> {
+  const fields: Record<string, string> = {}
+  const nameIndex = line.search(/(^|\s)name=/)
+  let rest = line
+  if (nameIndex !== -1) {
+    const nameStart = line.indexOf('name=', nameIndex) + 'name='.length
+    fields.name = line.slice(nameStart).trim()
+    rest = line.slice(0, nameIndex)
+  }
+  for (const match of rest.matchAll(/(\w+)=(\S+)/g)) {
+    fields[match[1]] = match[2]
+  }
+  return fields
+}
+
+function toInt(fields: Record<string, string>, key: string, required: true): number
+function toInt(fields: Record<string, string>, key: string, required?: false): number | undefined
+function toInt(fields: Record<string, string>, key: string, required = false) {
+  const raw = fields[key]
+  if (raw === undefined) {
+    if (required) throw new YencError(`Missing "${key}" in yEnc header`)
+    return undefined
+  }
+  const value = Number.parseInt(raw, 10)
+  if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`)
+  return value
+}
+
+function toCrc(fields: Record<string, string>, key: string) {
+  const raw = fields[key]
+  if (raw === undefined) return undefined
+  if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`)
+  return raw.toLowerCase().padStart(8, '0')
+}
+
+/**
+ * Number of parts in the file. Many posters leave `total=` out of `=ybegin`;
+ * then it follows from the part size, since all parts except the last one
+ * are the same size.
+ */
+export function inferTotalParts(meta: Pick<YencMeta, 'begin' | 'part'>): number {
+  const { begin, part } = meta
+  if (begin.total) return begin.total
+  if (!part || begin.part === undefined) return 1
+  if (part.end >= begin.size) return begin.part
+  return Math.ceil(begin.size / (part.end - part.begin + 1))
+}
+
+function lineAt(buffer: Buffer, start: number) {
+  const eol = buffer.indexOf(CRLF, start)
+  const end = eol === -1 ? buffer.length : eol
+  return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length }
+}
+
+/**
+ * Locates and parses the yEnc header/trailer lines of one article body,
+ * without decoding the data itself.
+ */
+export function parseYencMeta(body: Buffer): YencMeta {
+  const beginIndex = body.indexOf(BEGIN)
+  if (beginIndex === -1) throw new YencError('No =ybegin line found')
+
+  const beginLine = lineAt(body, beginIndex)
+  const beginFields = parseFields(beginLine.text)
+  const begin: YencBegin = {
+    line: toInt(beginFields, 'line', true),
+    size: toInt(beginFields, 'size', true),
+    name: beginFields.name ?? '',
+    part: toInt(beginFields, 'part'),
+    total: toInt(beginFields, 'total'),
+  }
+  if (!begin.name) throw new YencError('Missing "name" in =ybegin line')
+
+  let dataStart = beginLine.next
+  let part: YencPart | undefined
+  if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) {
+    const partLine = lineAt(body, dataStart)
+    const partFields = parseFields(partLine.text)
+    part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) }
+    dataStart = partLine.next
+  }
+
+  const endIndex = body.lastIndexOf(END)
+  if (endIndex === -1 || endIndex < dataStart - CRLF.length) {
+    throw new YencError('No =yend line found')
+  }
+  const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text)
+  const end: YencEnd = {
+    size: toInt(endFields, 'size', true),
+    part: toInt(endFields, 'part'),
+    pcrc32: toCrc(endFields, 'pcrc32'),
+    crc32: toCrc(endFields, 'crc32'),
+  }
+
+  return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) }
+}