import { explored, signatureKey, uploadTool } from './upload_tools.js' import { fileStem, looksRandom } from '../release/index.js' /** * What a worker drops, and why (REBUILD.md, rules and recovery). Sent with * every lease; `version` is recorded on the range, so a changed rule set * shows which ranges to read again. */ export interface Rules { version: number /** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */ priorities: Record /** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */ droppedSignatures: string[] /** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */ droppedTools: string[] /** A random-named file of at most this many parts: a random subject per article, needs bodies. */ perArticleMaxParts: number /** Share of each rule's drops kept in full as a sample (by a hash of the release key). */ samplePercent: number /** * Drop releases of bare executables (malware floods posted under film and * software names; 92% of the catalog rows of the first test ranges, LOG.md * 2026-10-09). Genuine software comes in archives. */ dropExecutables: boolean /** * Posters whose uploads are dropped whatever they are: fragments of the poster lowercased without punctuation * ("@__Celeb_Movie__@" → "celebmovie"). The CelebMovie family's scraped celebrity pictures, the same files again and * again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)). */ droppedPosters: string[] } export const DEFAULT_RULES: Omit = { version: 1, droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'], perArticleMaxParts: 3, samplePercent: 1, dropExecutables: true, droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'], } export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean } /** A poster as droppedPosters matches it: lowercased, letters and digits only. */ export function posterWords(poster: string): string { return poster.replace(/[^A-Za-z0-9]/g, '').toLowerCase() } /** Files that come along with a release's content: they don't make it one of executables or not. */ const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i const EXECUTABLE = /\.(exe|scr|com|pif|bat|cmd|vbs|lnk)\s*$/i export function isRandomFile(name: string) { return looksRandom(fileStem(name)) } /** * At least half the letters and digits of a file's stem in tokens of 5 or more that switch case (lower to upper) or * between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"): the names a * random-subject-per-article upload gives each article that looksRandom misses for their separators. */ export function isScrambledFile(name: string) { let total = 0 let scrambled = 0 for (const token of fileStem(name).split(/[^A-Za-z0-9]+/)) { total += token.length let switches = 0 for (let i = 1; i < token.length; i++) { const [a, b] = [token[i - 1], token[i]] if ((/[a-z]/.test(a) && /[A-Z]/.test(b)) || /\d/.test(a) !== /\d/.test(b)) switches++ } if (token.length >= 5 && switches >= 3) scrambled += token.length } return total >= 8 && scrambled * 2 >= total } /** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */ export function ownerOf(groups: string[], priorities: Record): string | null { let owner: string | null = null for (const g of groups) { const p = priorities[g] if (p !== undefined && (owner === null || p < priorities[owner])) owner = g } return owner } export class RuleSet { private signatures: Set private tools: Set private posters: string[] constructor(readonly rules: Rules) { this.posters = rules.droppedPosters ?? [] this.signatures = new Set(rules.droppedSignatures) this.tools = new Set(rules.droppedTools) } /** * A release's verdict from its files: dropped when its poster is; when its content is bare * executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is * dropped, as per-article random subjects when every file has few parts. */ judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict { const sample = explored(releaseKey, this.rules.samplePercent) if (this.posters.length > 0 && files.length > 0) { const words = posterWords(files[0].poster) const dropped = this.posters.find((p) => words.includes(p)) if (dropped) return { rule: 'poster', key: dropped, sample } } if (this.rules.dropExecutables) { const content = files.filter((f) => !SIDECAR.test(f.name)) if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) { return { rule: 'executable', key: EXECUTABLE.exec(content[0].name)![1].toLowerCase(), sample } } } if (files.some((f) => !isRandomFile(f.name))) return { rule: null } const first = files[0] const tool = uploadTool(first.firstMessageId) // Garbage first: a dropped tool or signature isn't worth reading bodies for either. const group = first.groups[0] ?? '' if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample } if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample } return { rule: null } } }