Rebuild: phase 1 modules from the current app, ordered by pipeline stage

lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys,
segments from overview lines; the yEnc path stays behind for phase 2),
lib/release (identity, random names, now in lib), lib/files (aggregator,
volume names), lib/rules (posting tools, owner by priority, release
verdicts), lib/chunks (release chunk codec). Their unit tests came along;
new ones for codec, rules and volumes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/rules/index.ts

@@ -0,0 +1,2 @@
+export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
+export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, type Rules, type Verdict } from './rule_set.js'

lib/rules/rule_set.ts

@@ -0,0 +1,69 @@
+import { explored, signatureKey, uploadTool } from './upload_tools.js'
+import { fileStem, looksRandom } from '../release/index.js'
+
+/**
+ * What a worker drops, and why (REBUILD.md, rules and recovery). Sent with
+ * every lease; `version` is recorded on the range, so a changed rule set
+ * shows which ranges to read again.
+ */
+export interface Rules {
+  version: number
+  /** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */
+  priorities: Record<string, number>
+  /** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */
+  droppedSignatures: string[]
+  /** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */
+  droppedTools: string[]
+  /** A random-named file of at most this many parts: a random subject per article, needs bodies. */
+  perArticleMaxParts: number
+  /** Share of each rule's drops kept in full as a sample (by a hash of the release key). */
+  samplePercent: number
+}
+
+export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
+  version: 1,
+  droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'],
+  perArticleMaxParts: 3,
+  samplePercent: 1,
+}
+
+export type Verdict = { rule: null } | { rule: 'garbage' | 'per-article'; key: string; sample: boolean }
+
+export function isRandomFile(name: string) {
+  return looksRandom(fileStem(name))
+}
+
+/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
+export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
+  let owner: string | null = null
+  for (const g of groups) {
+    const p = priorities[g]
+    if (p !== undefined && (owner === null || p < priorities[owner])) owner = g
+  }
+  return owner
+}
+
+export class RuleSet {
+  private signatures: Set<string>
+  private tools: Set<string>
+  constructor(readonly rules: Rules) {
+    this.signatures = new Set(rules.droppedSignatures)
+    this.tools = new Set(rules.droppedTools)
+  }
+
+  /**
+   * A release's verdict from its files: kept as a whole when any file has a
+   * readable name; else dropped as per-article random subjects when every
+   * file has few parts, as garbage when its signature or tool is dropped.
+   */
+  judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict {
+    if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
+    const first = files[0]
+    const tool = uploadTool(first.firstMessageId)
+    const sample = explored(releaseKey, this.rules.samplePercent)
+    if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample }
+    const group = first.groups[0] ?? ''
+    if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample }
+    return { rule: null }
+  }
+}

lib/rules/upload_tools.ts

@@ -0,0 +1,52 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * Upload signatures (CONCEPT.md D42): which tool posted an article, read from
+ * its Message-ID, and the group it was posted to first. Together they predict
+ * whether a random-named upload will ever get a name: ngPost uploads in
+ * a.b.ath almost never do, nyuu uploads in a.b.friends always do (LOG.md
+ * 2026-10-08 (5)).
+ */
+const TOOLS: [string, RegExp][] = [
+  // FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu: 12 random capital+small letter pairs, a ms timestamp
+  ['nyuu', /^(?:[A-Z][a-z]){12}-\d{13}@nyuu$/],
+  ['nyuu-host', /^(?:[A-Z][a-z]){12}-\d{13}@.+$/],
+  // b94268bc702040f7b811d5318f727e4f@ngPost: a UUID's hex
+  ['ngPost', /^[0-9a-f]{32}@ngPost$/],
+  ['yuun', /^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$/],
+  ['hex32', /^[0-9a-f]{32}@[A-Za-z0-9.-]+$/],
+  ['JBinUp', /@JBin(?:Up|Down)\.local$/],
+  // part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local
+  ['partNofM', /^part\d+of\d+\./i],
+  // 1790923856877$1729905757859$1@localhost
+  ['ts-dollar', /^\d{13}\$\d+\$\d+@localhost$/],
+  // t3Y4mV92YCp0McxDc5@zLZlsw.RYqes: random local part and random host
+  ['random', /^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$/],
+]
+
+/** The posting tool of a Message-ID (without angle brackets), or 'other'. */
+export function uploadTool(messageId: string | null | undefined): string {
+  if (!messageId) return 'other'
+  for (const [tool, pattern] of TOOLS) if (pattern.test(messageId)) return tool
+  return 'other'
+}
+
+/** The group of a signature: the first the upload was posted to, as stored in `groups`. */
+export function signatureGroup(groups: unknown): string {
+  return Array.isArray(groups) && typeof groups[0] === 'string' ? groups[0] : ''
+}
+
+/** The key of a signature in a Set. */
+export function signatureKey(tool: string, group: string) {
+  return `${tool}\t${group}`
+}
+
+/**
+ * Whether an upload is in the share of dropped signatures that is still kept
+ * (and probed), so their rates keep being measured. Decided per release key,
+ * so a release is kept or dropped as a whole.
+ */
+export function explored(key: string, percent: number): boolean {
+  if (percent <= 0) return false
+  return createHash('md5').update(key).digest().readUInt32BE(0) % 10_000 < percent * 100
+}