import { explored, signatureKey, uploadTool } from './upload_tools.js' import { fileStem, looksRandom } from '../release/index.js' /** * What a worker drops, and why (REBUILD.md, rules and recovery). Sent with * every lease; `version` is recorded on the range, so a changed rule set * shows which ranges to read again. */ export interface Rules { version: number /** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */ priorities: Record /** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */ droppedSignatures: string[] /** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */ droppedTools: string[] /** A random-named file of at most this many parts: a random subject per article, needs bodies. */ perArticleMaxParts: number /** Share of each rule's drops kept in full as a sample (by a hash of the release key). */ samplePercent: number } export const DEFAULT_RULES: Omit = { version: 1, droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'], perArticleMaxParts: 3, samplePercent: 1, } export type Verdict = { rule: null } | { rule: 'garbage' | 'per-article'; key: string; sample: boolean } export function isRandomFile(name: string) { return looksRandom(fileStem(name)) } /** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */ export function ownerOf(groups: string[], priorities: Record): string | null { let owner: string | null = null for (const g of groups) { const p = priorities[g] if (p !== undefined && (owner === null || p < priorities[owner])) owner = g } return owner } export class RuleSet { private signatures: Set private tools: Set constructor(readonly rules: Rules) { this.signatures = new Set(rules.droppedSignatures) this.tools = new Set(rules.droppedTools) } /** * A release's verdict from its files: kept as a whole when any file has a * readable name; else dropped as garbage when its signature or tool is * dropped, as per-article random subjects when every file has few parts. */ judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict { if (files.some((f) => !isRandomFile(f.name))) return { rule: null } const first = files[0] const tool = uploadTool(first.firstMessageId) const sample = explored(releaseKey, this.rules.samplePercent) // Garbage first: a dropped tool or signature isn't worth reading bodies for either. const group = first.groups[0] ?? '' if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample } if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample } return { rule: null } } }