Rebuild: phase 1 modules from the current app, ordered by pipeline stage

lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys,
segments from overview lines; the yEnc path stays behind for phase 2),
lib/release (identity, random names, now in lib), lib/files (aggregator,
volume names), lib/rules (posting tools, owner by priority, release
verdicts), lib/chunks (release chunk codec). Their unit tests came along;
new ones for codec, rules and volumes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/headers/file_key.ts

@@ -0,0 +1,47 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * Identity of one uploaded file: every segment of it has the same poster, the
+ * same subject apart from the segment counter, and the same segment count.
+ * The group is deliberately left out so crossposts resolve to one file.
+ * See CONCEPT.md (D12).
+ */
+export function fileKey(poster: string, base: string, totalSegments: number): string {
+  return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex')
+}
+
+/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */
+export function looksLikeFilename(name: string): boolean {
+  return /\.[A-Za-z0-9]{1,5}$/.test(name) && !/^[0-9a-f]{32,}$/i.test(name)
+}
+
+/**
+ * Identity of a file known only from its yEnc header (obfuscated posts).
+ * The poster is left out: obfuscating posters randomise it per article.
+ * - A real filename is stable across the articles of a file, and needed to
+ *   tell apart the equal-sized volumes of one archive.
+ * - A random, extensionless name can change per article too; the exact
+ *   byte size and part count are what stays the same.
+ * Collisions are caught by the conflict check in IngestService.
+ */
+export function weakFileKey(name: string, size: number, totalParts: number): string {
+  const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
+  return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
+}
+
+/**
+ * Identity of a file by its counters, for uploads that give every article
+ * its own random subject (CONCEPT.md D36): poster, file counter `[i/M]`,
+ * part count and posting day stay the same across the articles of one file.
+ * The day keeps different uploads with the same shape apart; collisions that
+ * remain are caught by the conflict check (these files are weak identities).
+ * Null when the subject has no file counter.
+ */
+export function counterFileKey(segment: { poster: string; subject: string; totalParts: number; postedAt?: Date }): string | null {
+  const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject)
+  if (!counter || segment.totalParts < 2) return null
+  const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : ''
+  return createHash('sha1')
+    .update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
+    .digest('hex')
+}

lib/headers/index.ts

@@ -0,0 +1,3 @@
+export { parseSubject, type ParsedSubject } from './subject.js'
+export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
+export { segmentFromHeader, type IngestSegment, type OverviewHeader } from './segment.js'

lib/headers/segment.ts

@@ -0,0 +1,72 @@
+import { parseXrefGroups } from '../nntp/xref.js'
+import { fileKey } from './file_key.js'
+import { parseSubject } from './subject.js'
+import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
+
+/** One segment, with the identity of the file it belongs to already resolved. */
+export interface IngestSegment {
+  fileKey: string
+  poster: string
+  /** Subject without segment counter. */
+  subject: string
+  name: string
+  totalParts: number
+  number: number
+  /** Without angle brackets. */
+  messageId: string
+  bytes: number
+  articleNumber?: bigint | number
+  postedAt?: Date
+  groups: string[]
+  /**
+   * The identity comes from the yEnc name only (obfuscated posts). Such names
+   * can be shared by several equal-sized files of one post, so a second
+   * article for the same part number marks the binary as conflicted.
+   */
+  weakIdentity?: boolean
+  /** Release (download) this file belongs to, when it can be determined. */
+  release?: ReleaseIdentity | null
+  /** Exact file size from =ybegin (body-derived segments only). */
+  size?: number
+  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
+  pcrc32?: string
+  partSize?: number
+  fileCrc32?: string
+  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
+  md5_16k?: string
+}
+
+/** An XOVER overview entry as returned by nntp-js. */
+export type OverviewHeader = Record<string, string>
+
+function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
+  const xrefGroups = parseXrefGroups(header.xref)
+  return {
+    poster: header.from ?? '',
+    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
+    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
+    articleNumber,
+    postedAt: header.date ? new Date(header.date) : undefined,
+    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
+  }
+}
+
+/** Builds a segment from the subject line; null when the subject is obfuscated. */
+export function segmentFromHeader(
+  header: OverviewHeader,
+  groupName: string,
+  articleNumber?: bigint | number
+): IngestSegment | null {
+  const parsed = header.subject ? parseSubject(header.subject) : null
+  if (!parsed || !header['message-id']) return null
+  const base = common(header, groupName, articleNumber)
+  return {
+    ...base,
+    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
+    subject: parsed.base,
+    name: parsed.name,
+    totalParts: parsed.totalSegments,
+    number: parsed.segment,
+    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
+  }
+}

lib/headers/subject.ts

@@ -0,0 +1,59 @@
+export interface ParsedSubject {
+  /** Filename from the quoted part of the subject. */
+  name: string
+  /** Segment number `k` from `(k/N)`. */
+  segment: number
+  /** Segment count `N` from `(k/N)`. */
+  totalSegments: number
+  /** File counter `[i/M]`, when present. */
+  fileIndex?: number
+  fileTotal?: number
+  /**
+   * The subject with the segment counter removed. Identical for every segment
+   * of one uploaded file, so it is part of the file identity (see fileKey).
+   */
+  base: string
+}
+
+const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
+const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
+
+/**
+ * Parses a binary post subject such as
+ * `[07/45] - "Some.Release.part06.rar" yEnc (12/137) 51315352`.
+ *
+ * Returns null for subjects without a quoted filename and segment counter
+ * (obfuscated posts), which need their yEnc header read from the body instead.
+ */
+export function parseSubject(subject: string): ParsedSubject | null {
+  const nameEnd = subject.lastIndexOf('"')
+  const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
+  if (nameStart === -1) return null
+  const name = subject.slice(nameStart + 1, nameEnd)
+  if (!name) return null
+
+  // The segment counter is the last "(k/N)" after the filename; a file size may follow it.
+  let counter: RegExpExecArray | undefined
+  for (const match of subject.matchAll(SEGMENT_COUNTER)) {
+    if (match.index > nameEnd) counter = match
+  }
+  if (!counter) return null
+
+  const segment = Number.parseInt(counter[1], 10)
+  const totalSegments = Number.parseInt(counter[2], 10)
+  if (totalSegments < 1 || segment < 1 || segment > totalSegments) return null
+
+  const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length))
+    .replace(/\s+/g, ' ')
+    .trim()
+
+  const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart))
+  return {
+    name,
+    segment,
+    totalSegments,
+    fileIndex: fileCounter ? Number.parseInt(fileCounter[1], 10) : undefined,
+    fileTotal: fileCounter ? Number.parseInt(fileCounter[2], 10) : undefined,
+    base,
+  }
+}