Rebuild: phase 2's prober reads the probe queue

probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims
waiting candidates of settled releases (lone, then par2, then archive),
fetches each article with BODY over connections of its own (no
compression) and inspects it: the yEnc name and size, the content type
from the first bytes, a par2 set's files, a RAR volume's entries and
whether they're encrypted. Results per release and kind in
release_inspections (CHUNK_TABLESPACE, the HDD); on the release
passworded, inner_name (the largest content name) and inspected_at.
430 = missing, not yEnc = error, both final; connection trouble puts
the candidate back.

Fetch and parse in one process: an article takes ~0.1 s per connection,
parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc,
lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body;
yencode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/content/index.ts

@@ -0,0 +1,2 @@
+/** Content recognition by magic bytes (from voyager 1's lib/content). */
+export { sniffContentType, type ContentType } from './sniff.js'

lib/content/sniff.ts

@@ -0,0 +1,81 @@
+/**
+ * Recognises a file type from its first bytes, independent of AdonisJS.
+ * `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
+ * "nzbe", 16 random bytes (salt/IV), then ciphertext.
+ * `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com`
+ * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
+ */
+export type ContentType =
+  /** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */
+  | 'missing'
+  | 'nzbe'
+  | 'par2'
+  | 'rar4'
+  | 'rar5'
+  | '7z'
+  | 'zip'
+  | 'zstd'
+  | 'gzip'
+  | 'xz'
+  | 'matroska'
+  | 'mp4'
+  | 'rclone'
+  | 'yencbin'
+  | 'iso'
+  | 'udf'
+  | 'pdf'
+  | 'nzb'
+  | 'jpeg'
+  | 'png'
+  | 'mp3'
+  | 'flac'
+  | 'avi'
+  | 'wav'
+  | 'ogg'
+  | 'bzip2'
+
+/** Magic bytes at an offset; null matches any byte. */
+const SIGNATURES: [ContentType, number, (number | null)[]][] = [
+  // Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…):
+  // the bytes they share.
+  ['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]],
+  ['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
+  ['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
+  ['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
+  ['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]],
+  ['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]],
+  ['zip', 0, [0x50, 0x4b, 0x03, 0x04]],
+  ['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]],
+  ['gzip', 0, [0x1f, 0x8b]],
+  ['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]],
+  ['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]],
+  ['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box
+  ['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt
+  ['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]],
+  ['jpeg', 0, [0xff, 0xd8, 0xff]],
+  ['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]],
+  ['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag
+  ['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC"
+  ['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS"
+  ['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh"
+]
+
+export function sniffContentType(data: Buffer): ContentType | null {
+  for (const [type, offset, bytes] of SIGNATURES) {
+    if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type
+  }
+  // Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
+  // ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.
+  if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso'
+  if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' &&
+      /^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf'
+  // RIFF container: AVI or WAV by form type.
+  if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') {
+    const form = data.subarray(8, 12).toString('latin1')
+    if (form === 'AVI ') return 'avi'
+    if (form === 'WAVE') return 'wav'
+  }
+  const head = data.subarray(0, 512).toString('latin1')
+  if (/^\s*(<\?xml[^>]*>\s*)?(<!DOCTYPE nzb|<nzb[\s>])/.test(head)) return 'nzb'
+  return null
+}

lib/nntp/index.ts

@@ -1,4 +1,4 @@
-export { XoverClient, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
+export { XoverClient, MissingArticle, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
 export { parseXrefGroups } from './xref.js'
 export { parseGroupResponse, type GroupInfo } from './group.js'
 export { hedge, type Request, type Hedged } from './hedge.js'

lib/nntp/xover_client.ts

@@ -34,6 +34,9 @@ const DOT_LINE = Buffer.from('.\r\n')
  * followed by a plain "." line; an empty range is a 420 without body.
  * About six times less traffic per header (LOG.md 2026-10-05).
  */
+/** The server doesn't have the article (430): taken down or expired. */
+export class MissingArticle extends Error {}
+
 export class XoverClient {
   private chunks: Buffer[] = []
   private length = 0
@@ -228,6 +231,37 @@ export class XoverClient {
     }
   }
 
+  /**
+   * An article's body by message-id, raw (dot-stuffed, "\r\n.\r\n" cut off):
+   * yEnc decoding undoes the stuffing. A connection without COMPRESS GZIP.
+   * 430 (no such article) throws MissingArticle.
+   */
+  async body(messageId: string): Promise<Buffer> {
+    return this.timed(async () => {
+      const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
+      const reply = await this.command(`BODY ${id}`)
+      if (reply.startsWith('430') || reply.startsWith('423')) throw new MissingArticle(reply)
+      await this.expect(reply, ['222'])
+      return this.rawBody()
+    })
+  }
+
+  /** A multi-line body up to "\r\n.\r\n", as bytes, the terminator cut off. */
+  private async rawBody(): Promise<Buffer> {
+    let searched = 0
+    while (true) {
+      const all = this.buffer()
+      if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) {
+        this.take(3)
+        return Buffer.alloc(0)
+      }
+      const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
+      if (at !== -1) return Buffer.from(this.take(at + TERMINATOR.length).subarray(0, at + 2))
+      searched = all.length
+      await this.more()
+    }
+  }
+
   /** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */
   private async plainBody(): Promise<string> {
     let searched = 0

lib/par2/index.ts

@@ -0,0 +1,2 @@
+/** Par2 FileDesc packets (from voyager 1's lib/par2): the files a par2 set covers. */
+export { parsePar2, isPar2, type Par2File, type Par2Set } from './parse.js'

lib/par2/parse.ts

@@ -0,0 +1,74 @@
+import { createHash } from 'node:crypto'
+
+export interface Par2File {
+  /** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */
+  fileId: string
+  /** MD5 of the whole file (hex). */
+  md5: string
+  /** MD5 of the first 16 KiB (hex). */
+  md5_16k: string
+  length: number
+  name: string
+}
+
+export interface Par2Set {
+  /** Recovery set ID (hex), shared by every packet of one par2 set. */
+  recoverySetId: string
+  files: Par2File[]
+}
+
+const MAGIC = Buffer.from('PAR2\0PKT', 'latin1')
+const HEADER_SIZE = 64
+const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1')
+
+export function isPar2(data: Buffer): boolean {
+  return data.subarray(0, MAGIC.length).equals(MAGIC)
+}
+
+/**
+ * Reads the FileDesc packets of a par2 file (index or recovery volume).
+ * Packets with a bad length or checksum are skipped; parsing resumes at the
+ * next magic sequence. Returns null when no valid FileDesc packet is found.
+ */
+export function parsePar2(data: Buffer): Par2Set | null {
+  const files = new Map<string, Par2File>()
+  let recoverySetId: string | undefined
+  let offset = data.indexOf(MAGIC)
+
+  while (offset !== -1 && offset + HEADER_SIZE <= data.length) {
+    const length = Number(data.readBigUInt64LE(offset + 8))
+    const valid =
+      length >= HEADER_SIZE &&
+      length % 4 === 0 &&
+      offset + length <= data.length &&
+      createHash('md5')
+        .update(data.subarray(offset + 32, offset + length))
+        .digest()
+        .equals(data.subarray(offset + 16, offset + 32))
+
+    if (!valid) {
+      offset = data.indexOf(MAGIC, offset + 1)
+      continue
+    }
+
+    const setId = data.subarray(offset + 32, offset + 48).toString('hex')
+    const type = data.subarray(offset + 48, offset + 64)
+    if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) {
+      recoverySetId = setId
+      const body = data.subarray(offset + HEADER_SIZE, offset + length)
+      const fileId = body.subarray(0, 16).toString('hex')
+      files.set(fileId, {
+        fileId,
+        md5: body.subarray(16, 32).toString('hex'),
+        md5_16k: body.subarray(32, 48).toString('hex'),
+        length: Number(body.readBigUInt64LE(48)),
+        // The name is padded with NULs to a multiple of 4 bytes.
+        name: body.subarray(56).toString('utf8').replace(/\0+$/, ''),
+      })
+    }
+    offset += length
+  }
+
+  if (!recoverySetId || files.size === 0) return null
+  return { recoverySetId, files: [...files.values()] }
+}

lib/probe/index.ts

@@ -0,0 +1 @@
+export { inspect, innerName, MAX_NAMES, type Inspection } from './inspect.js'

lib/probe/inspect.ts

@@ -0,0 +1,79 @@
+import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
+import { isPar2, parsePar2 } from '#lib/par2/index'
+import { parseRarHeaders } from '#lib/rar/index'
+import { sniffContentType, type ContentType } from '#lib/content/index'
+
+/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
+export interface Inspection {
+  /** The file name in the yEnc header. */
+  yencName: string
+  /** The whole file's size, from the yEnc header. */
+  size: number
+  type: ContentType | null
+  /** Names inside: an archive's entries or a par2 set's files, the largest first, at most MAX_NAMES. */
+  names?: string[]
+  /** How many entries or files there are in all. */
+  entries?: number
+  /** An archive with encrypted entries or headers. */
+  passworded?: boolean
+}
+
+/** Names kept per inspection: the catalog wants the content's name, not every volume's. */
+export const MAX_NAMES = 8
+const MAX_NAME_LENGTH = 200
+
+/** Archive volumes, par2 volumes and split parts: names of the packaging, not the content. */
+const PACKAGING = /\.(rar|r\d{2,3}|s\d{2}|\d{3}|7z(\.\d+)?|zip|z\d{2}|par2|sfv|nfo|nzb)$/i
+
+/** Decodes a body; some tools declare a size that leaves out a header, the CRC still has to match. */
+function decodeBody(body: Buffer): DecodedSegment {
+  try {
+    return decodeSegment(body)
+  } catch (error) {
+    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+    return decodeSegment(body, { lenientSize: true })
+  }
+}
+
+function largestFirst(items: { name: string; size: number }[]): string[] {
+  return items
+    .sort((a, b) => b.size - a.size)
+    .slice(0, MAX_NAMES)
+    .map((i) => i.name.slice(0, MAX_NAME_LENGTH))
+}
+
+/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */
+export function inspect(body: Buffer): Inspection {
+  const segment = decodeBody(body)
+  const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
+  if (isPar2(segment.data)) {
+    // An index file fits one article; what a larger one cut off is skipped by the parser.
+    const set = parsePar2(segment.data)
+    if (set) {
+      result.entries = set.files.length
+      result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
+    }
+    return result
+  }
+  if (result.type === 'rar4' || result.type === 'rar5') {
+    const info = parseRarHeaders(segment.data)
+    if (info) {
+      const files = info.entries.filter((e) => !e.directory)
+      result.entries = files.length
+      result.names = largestFirst(files.map((e) => ({ name: e.name, size: Number(e.size ?? 0) })))
+      result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
+    }
+  }
+  return result
+}
+
+/**
+ * A release's best inner name from its inspections: the largest named entry
+ * of its archive, else the largest file of its par2 set that isn't
+ * packaging, else a lone file's yEnc name. Null when none says more.
+ */
+export function innerName(inspections: Partial<Record<string, Inspection>>): string | null {
+  const content = (i?: Inspection) => i?.names?.find((n) => !PACKAGING.test(n))
+  const pick = content(inspections.archive) ?? content(inspections.par2) ?? (inspections.lone?.yencName || undefined)
+  return pick ? pick.replace(/^.*[\\/]/, '').slice(0, 255) : null
+}

lib/rar/index.ts

@@ -0,0 +1,2 @@
+/** RAR 4 and 5 headers (from voyager 1's lib/rar): a volume's listing, encryption, volume flags. */
+export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'

lib/rar/parse.ts

@@ -0,0 +1,351 @@
+import { crc32 } from 'node:zlib'
+
+export interface RarEntry {
+  name: string
+  /** Unpacked size in bytes (null when the archiver did not record it). */
+  size: number | null
+  packedSize: number
+  directory: boolean
+  /** The file's data is password protected. */
+  encrypted: boolean
+  /** Continues from the previous volume. */
+  splitBefore: boolean
+  /** Continues in the next volume. */
+  splitAfter: boolean
+  /** Stored without compression, so the data can be copied out as is. */
+  stored: boolean
+  /** Offset of the entry's (packed) data in the parsed buffer. */
+  dataOffset: number
+}
+
+export interface RarInfo {
+  /** 7z listings take the same shape (see lib/sevenzip). */
+  format: 'rar4' | 'rar5' | '7z'
+  /** Part of a multi-volume set. */
+  volume: boolean
+  /** First volume of the set (null when the format version does not say). */
+  firstVolume: boolean | null
+  solid: boolean
+  /** File names are encrypted too; nothing can be listed without the password. */
+  headersEncrypted: boolean
+  entries: RarEntry[]
+  /** The buffer ended before the end-of-archive header (normal for one segment). */
+  truncated: boolean
+  /**
+   * Offset of the first block that was not read: where the next header
+   * starts (possibly beyond the buffer, after an entry's data), or the
+   * header the buffer cut off. After the end-of-archive header, its end.
+   */
+  next: number
+}
+
+const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
+const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
+
+export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
+  if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5'
+  if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4'
+  return null
+}
+
+/**
+ * Lists a RAR archive from its headers only, so the first segment of a first
+ * volume is enough. Returns null when the data is not a RAR archive.
+ *
+ * With `continueAs`, `data` starts at a header in the middle of a volume (no
+ * signature), e.g. where a previous parse said the next header is.
+ */
+export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
+  if (continueAs === 'rar4') return parseRar4(data, 0)
+  if (continueAs === 'rar5') return parseRar5(data, 0)
+  const format = rarFormat(data)
+  if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
+  if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
+  return null
+}
+
+// --- RAR 4.x (RAR 1.5 – 4.x archive format) -------------------------------
+
+const R4_MAIN = 0x73
+const R4_FILE = 0x74
+const R4_END = 0x7b
+const R4_LONG_BLOCK = 0x8000
+
+function parseRar4(data: Buffer, start: number): RarInfo {
+  const info: RarInfo = {
+    format: 'rar4',
+    volume: false,
+    firstVolume: null,
+    solid: false,
+    headersEncrypted: false,
+    entries: [],
+    truncated: true,
+    next: start,
+  }
+  let p = start
+
+  while (p + 7 <= data.length) {
+    const headCrc = data.readUInt16LE(p)
+    const type = data[p + 2]
+    const flags = data.readUInt16LE(p + 3)
+    const size = data.readUInt16LE(p + 5)
+    if (size < 7 || p + size > data.length) break
+
+    if (type === R4_END) {
+      info.truncated = false
+      p += size
+      break
+    }
+
+    let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0)
+
+    if (type === R4_MAIN) {
+      info.volume = Boolean(flags & 0x0001)
+      info.solid = Boolean(flags & 0x0008)
+      info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true
+      if (flags & 0x0080) {
+        info.headersEncrypted = true
+        break
+      }
+    } else if (type === R4_FILE) {
+      if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break
+      const large = Boolean(flags & 0x0100)
+      let packed = data.readUInt32LE(p + 7)
+      let unpacked = data.readUInt32LE(p + 11)
+      const nameSize = data.readUInt16LE(p + 26)
+      let nameStart = p + 32
+      if (large) {
+        packed += data.readUInt32LE(p + 32) * 2 ** 32
+        unpacked += data.readUInt32LE(p + 36) * 2 ** 32
+        nameStart = p + 40
+      }
+      const rawName = data.subarray(nameStart, nameStart + nameSize)
+      info.entries.push({
+        name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'),
+        // 0xffffffff with LHD_LARGE unset means "unknown" in some archivers.
+        size: unpacked === 0xffffffff && !large ? null : unpacked,
+        packedSize: packed,
+        directory: (flags & 0x00e0) === 0x00e0,
+        encrypted: Boolean(flags & 0x0004),
+        splitBefore: Boolean(flags & 0x0001),
+        splitAfter: Boolean(flags & 0x0002),
+        stored: data[p + 25] === 0x30,
+        dataOffset: p + size,
+      })
+      next = p + size + packed
+    }
+
+    if (next <= p) break
+    p = next
+  }
+  // Archivers before RAR 3.0 don't set the first-volume flag; a first
+  // volume is one whose first file does not continue from a previous one.
+  if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
+    info.firstVolume = true
+  }
+  info.next = p
+  return info
+}
+
+/**
+ * RAR 4 stores Unicode names as "ASCII\0<compressed UTF-16>", where the
+ * compressed part refers back to the ASCII bytes (unrar's DecodeFileName).
+ */
+export function decodeRar4Name(raw: Buffer): string {
+  const zero = raw.indexOf(0)
+  if (zero === -1) return raw.toString('utf8')
+  const ascii = raw.subarray(0, zero)
+  const enc = raw.subarray(zero + 1)
+  if (enc.length === 0) return ascii.toString('latin1')
+
+  const out: number[] = []
+  const highByte = enc[0]
+  let encPos = 1
+  let flags = 0
+  let flagBits = 0
+  while (encPos < enc.length) {
+    if (flagBits === 0) {
+      flags = enc[encPos++]
+      flagBits = 8
+    }
+    switch (flags >> 6) {
+      case 0:
+        out.push(enc[encPos++])
+        break
+      case 1:
+        out.push(enc[encPos++] + (highByte << 8))
+        break
+      case 2:
+        out.push(enc[encPos] + (enc[encPos + 1] << 8))
+        encPos += 2
+        break
+      case 3: {
+        let length = enc[encPos++]
+        if (length & 0x80) {
+          const correction = enc[encPos++]
+          for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) {
+            out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8))
+          }
+        } else {
+          for (length += 2; length > 0 && out.length < ascii.length; length--) {
+            out.push(ascii[out.length])
+          }
+        }
+        break
+      }
+    }
+    flags = (flags << 2) & 0xff
+    flagBits -= 2
+  }
+  return String.fromCharCode(...out)
+}
+
+// --- RAR 5.0 ----------------------------------------------------------------
+
+const R5_MAIN = 1
+const R5_FILE = 2
+const R5_ENCRYPTION = 4
+const R5_END = 5
+
+/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */
+function vint(data: Buffer, offset: number): { value: number; next: number } | null {
+  let value = 0
+  let shift = 1
+  for (let i = offset; i < data.length && i < offset + 10; i++) {
+    value += (data[i] & 0x7f) * shift
+    shift *= 128
+    if (!(data[i] & 0x80)) return { value, next: i + 1 }
+  }
+  return null
+}
+
+function parseRar5(data: Buffer, start: number): RarInfo {
+  const info: RarInfo = {
+    format: 'rar5',
+    volume: false,
+    firstVolume: null,
+    solid: false,
+    headersEncrypted: false,
+    entries: [],
+    truncated: true,
+    next: start,
+  }
+  let p = start
+
+  while (p + 5 <= data.length) {
+    const headCrc = data.readUInt32LE(p)
+    const headerSize = vint(data, p + 4)
+    if (!headerSize || headerSize.value === 0) break
+    const headerEnd = headerSize.next + headerSize.value
+    if (headerEnd > data.length) break
+    if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break
+
+    const type = vint(data, headerSize.next)
+    const flags = type && vint(data, type.next)
+    if (!type || !flags) break
+    let cursor = flags.next
+    let extraSize = 0
+    let dataSize = 0
+    if (flags.value & 0x0001) {
+      const v = vint(data, cursor)
+      if (!v) break
+      extraSize = v.value
+      cursor = v.next
+    }
+    if (flags.value & 0x0002) {
+      const v = vint(data, cursor)
+      if (!v) break
+      dataSize = v.value
+      cursor = v.next
+    }
+
+    if (type.value === R5_END) {
+      info.truncated = false
+      p = headerEnd
+      break
+    }
+    if (type.value === R5_ENCRYPTION) {
+      info.headersEncrypted = true
+      break
+    }
+
+    if (type.value === R5_MAIN) {
+      const archiveFlags = vint(data, cursor)
+      if (!archiveFlags) break
+      info.volume = Boolean(archiveFlags.value & 0x0001)
+      // A volume number is only stored for volumes after the first.
+      info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true
+      info.solid = Boolean(archiveFlags.value & 0x0004)
+    } else if (type.value === R5_FILE) {
+      const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize)
+      if (!entry) break
+      info.entries.push(entry)
+    }
+
+    const next = headerEnd + dataSize
+    if (next <= p) break
+    p = next
+  }
+  info.next = p
+  return info
+}
+
+function parseRar5File(
+  data: Buffer,
+  start: number,
+  extraStart: number,
+  headerEnd: number,
+  headerFlags: number,
+  dataSize: number
+): RarEntry | null {
+  const fileFlags = vint(data, start)
+  const unpacked = fileFlags && vint(data, fileFlags.next)
+  const attributes = unpacked && vint(data, unpacked.next)
+  if (!fileFlags || !unpacked || !attributes) return null
+  let cursor = attributes.next
+  if (fileFlags.value & 0x0002) cursor += 4 // mtime
+  if (fileFlags.value & 0x0004) cursor += 4 // data CRC32
+  const compression = vint(data, cursor)
+  const hostOs = compression && vint(data, compression.next)
+  const nameLength = hostOs && vint(data, hostOs.next)
+  if (!compression || !hostOs || !nameLength) return null
+  const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8')
+  // Compression information: bits 7-9 are the method, 0 is "store".
+  const method = Math.floor(compression.value / 128) & 0x7
+
+  // Extra area records: size, type, data. Type 1 is file encryption.
+  let encrypted = false
+  let e = extraStart
+  while (e < headerEnd) {
+    const recordSize = vint(data, e)
+    const recordType = recordSize && vint(data, recordSize.next)
+    if (!recordSize || !recordType || recordSize.value === 0) break
+    if (recordType.value === 0x01) encrypted = true
+    e = recordSize.next + recordSize.value
+  }
+
+  return {
+    name,
+    size: fileFlags.value & 0x0008 ? null : unpacked.value,
+    packedSize: dataSize,
+    directory: Boolean(fileFlags.value & 0x0001),
+    encrypted,
+    splitBefore: Boolean(headerFlags & 0x0008),
+    splitAfter: Boolean(headerFlags & 0x0010),
+    stored: method === 0,
+    dataOffset: headerEnd,
+  }
+}
+
+/**
+ * The data of an entry stored without compression, copied out of the
+ * archive buffer. Null when it is compressed, encrypted, split over volumes
+ * or not completely in the buffer.
+ */
+export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null {
+  if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
+  if (entry.size === null || entry.size !== entry.packedSize) return null
+  const end = entry.dataOffset + entry.packedSize
+  if (end > data.length) return null
+  return data.subarray(entry.dataOffset, end)
+}

lib/yenc/decode.ts

@@ -0,0 +1,76 @@
+import yencode from 'yencode'
+import { YencCrcError, YencError } from './errors.js'
+import { parseYencMeta, type YencMeta } from './meta.js'
+
+export interface DecodedSegment {
+  meta: YencMeta
+  /** Decoded bytes of this segment. */
+  data: Buffer
+  /** 0-based offset of `data` within the complete file. */
+  offset: number
+}
+
+export interface DecodeOptions {
+  /**
+   * Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
+   * bodies, so this defaults to true.
+   */
+  stripDots?: boolean
+  /** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
+  verifyCrc?: boolean
+  /**
+   * Accept a decoded length that differs from the declared size (the CRC is
+   * still checked). Some posting tools declare a size that leaves out a
+   * header, e.g. the 20-byte header of `nzbe` containers.
+   */
+  lenientSize?: boolean
+}
+
+export function crc32Hex(data: Buffer): string {
+  return yencode.crc32(data).toString('hex')
+}
+
+/**
+ * Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
+ * without the data itself.
+ */
+export function combineCrc32(parts: { crc: string; length: number }[]): string {
+  let combined: Buffer | undefined
+  for (const part of parts) {
+    const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
+    combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
+  }
+  return (combined ?? Buffer.alloc(4)).toString('hex')
+}
+
+/**
+ * Decodes one yEnc article body and checks its size and CRC32 against the
+ * `=yend` trailer.
+ */
+export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
+  const { stripDots = true, verifyCrc = true, lenientSize = false } = options
+  const meta = parseYencMeta(body)
+  const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
+
+  if (lenientSize) {
+    // Only the CRC below decides.
+  } else if (data.length !== meta.end.size) {
+    throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
+  }
+  if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
+    throw new YencError(
+      `Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
+    )
+  }
+
+  if (verifyCrc) {
+    // Multi-part posts carry pcrc32 per part; single-part posts only crc32.
+    const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
+    if (expected) {
+      const actual = crc32Hex(data)
+      if (actual !== expected) throw new YencCrcError(expected, actual)
+    }
+  }
+
+  return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
+}

lib/yenc/errors.ts

@@ -0,0 +1,16 @@
+export class YencError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'YencError'
+  }
+}
+
+export class YencCrcError extends YencError {
+  constructor(
+    public readonly expected: string,
+    public readonly actual: string
+  ) {
+    super(`CRC32 mismatch: expected ${expected}, got ${actual}`)
+    this.name = 'YencCrcError'
+  }
+}

lib/yenc/index.ts

@@ -0,0 +1,4 @@
+/** yEnc article bodies (from voyager 1's lib/yenc): header and trailer, decoding with size and CRC checks. */
+export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js'
+export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js'
+export { YencError, YencCrcError } from './errors.js'

lib/yenc/meta.ts

@@ -0,0 +1,140 @@
+import { YencError } from './errors.js'
+
+/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */
+export interface YencBegin {
+  line: number
+  size: number
+  name: string
+  part?: number
+  total?: number
+}
+
+/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */
+export interface YencPart {
+  begin: number
+  end: number
+}
+
+/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */
+export interface YencEnd {
+  size: number
+  part?: number
+  pcrc32?: string
+  crc32?: string
+}
+
+export interface YencMeta {
+  begin: YencBegin
+  part?: YencPart
+  end: YencEnd
+  /** Byte range of the encoded data inside the article body. */
+  dataStart: number
+  dataEnd: number
+}
+
+const CRLF = Buffer.from('\r\n')
+const BEGIN = Buffer.from('=ybegin ')
+const PART = Buffer.from('=ypart ')
+const END = Buffer.from('\r\n=yend ')
+
+/**
+ * Parses `key=value` pairs from a yEnc header line. `name` is always the last
+ * field and runs to the end of the line, because filenames may contain spaces
+ * and `=`.
+ */
+function parseFields(line: string): Record<string, string> {
+  const fields: Record<string, string> = {}
+  const nameIndex = line.search(/(^|\s)name=/)
+  let rest = line
+  if (nameIndex !== -1) {
+    const nameStart = line.indexOf('name=', nameIndex) + 'name='.length
+    fields.name = line.slice(nameStart).trim()
+    rest = line.slice(0, nameIndex)
+  }
+  for (const match of rest.matchAll(/(\w+)=(\S+)/g)) {
+    fields[match[1]] = match[2]
+  }
+  return fields
+}
+
+function toInt(fields: Record<string, string>, key: string, required: true): number
+function toInt(fields: Record<string, string>, key: string, required?: false): number | undefined
+function toInt(fields: Record<string, string>, key: string, required = false) {
+  const raw = fields[key]
+  if (raw === undefined) {
+    if (required) throw new YencError(`Missing "${key}" in yEnc header`)
+    return undefined
+  }
+  const value = Number.parseInt(raw, 10)
+  if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`)
+  return value
+}
+
+function toCrc(fields: Record<string, string>, key: string) {
+  const raw = fields[key]
+  if (raw === undefined) return undefined
+  if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`)
+  return raw.toLowerCase().padStart(8, '0')
+}
+
+/**
+ * Number of parts in the file. Many posters leave `total=` out of `=ybegin`;
+ * then it follows from the part size, since all parts except the last one
+ * are the same size.
+ */
+export function inferTotalParts(meta: Pick<YencMeta, 'begin' | 'part'>): number {
+  const { begin, part } = meta
+  if (begin.total) return begin.total
+  if (!part || begin.part === undefined) return 1
+  if (part.end >= begin.size) return begin.part
+  return Math.ceil(begin.size / (part.end - part.begin + 1))
+}
+
+function lineAt(buffer: Buffer, start: number) {
+  const eol = buffer.indexOf(CRLF, start)
+  const end = eol === -1 ? buffer.length : eol
+  return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length }
+}
+
+/**
+ * Locates and parses the yEnc header/trailer lines of one article body,
+ * without decoding the data itself.
+ */
+export function parseYencMeta(body: Buffer): YencMeta {
+  const beginIndex = body.indexOf(BEGIN)
+  if (beginIndex === -1) throw new YencError('No =ybegin line found')
+
+  const beginLine = lineAt(body, beginIndex)
+  const beginFields = parseFields(beginLine.text)
+  const begin: YencBegin = {
+    line: toInt(beginFields, 'line', true),
+    size: toInt(beginFields, 'size', true),
+    name: beginFields.name ?? '',
+    part: toInt(beginFields, 'part'),
+    total: toInt(beginFields, 'total'),
+  }
+  if (!begin.name) throw new YencError('Missing "name" in =ybegin line')
+
+  let dataStart = beginLine.next
+  let part: YencPart | undefined
+  if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) {
+    const partLine = lineAt(body, dataStart)
+    const partFields = parseFields(partLine.text)
+    part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) }
+    dataStart = partLine.next
+  }
+
+  const endIndex = body.lastIndexOf(END)
+  if (endIndex === -1 || endIndex < dataStart - CRLF.length) {
+    throw new YencError('No =yend line found')
+  }
+  const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text)
+  const end: YencEnd = {
+    size: toInt(endFields, 'size', true),
+    part: toInt(endFields, 'part'),
+    pcrc32: toCrc(endFields, 'pcrc32'),
+    crc32: toCrc(endFields, 'crc32'),
+  }
+
+  return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) }
+}