107 lines
4.8 KiB
TypeScript
107 lines
4.8 KiB
TypeScript
import { explored, signatureKey, uploadTool } from './upload_tools.js'
|
|
import { fileStem, looksRandom } from '../release/index.js'
|
|
|
|
/**
|
|
* What a worker drops, and why (REBUILD.md, rules and recovery). Sent with
|
|
* every lease; `version` is recorded on the range, so a changed rule set
|
|
* shows which ranges to read again.
|
|
*/
|
|
export interface Rules {
|
|
version: number
|
|
/** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */
|
|
priorities: Record<string, number>
|
|
/** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */
|
|
droppedSignatures: string[]
|
|
/** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */
|
|
droppedTools: string[]
|
|
/** A random-named file of at most this many parts: a random subject per article, needs bodies. */
|
|
perArticleMaxParts: number
|
|
/** Share of each rule's drops kept in full as a sample (by a hash of the release key). */
|
|
samplePercent: number
|
|
/**
|
|
* Drop releases of bare executables (malware floods posted under film and
|
|
* software names; 92% of the catalog rows of the first test ranges, LOG.md
|
|
* 2026-10-09). Genuine software comes in archives.
|
|
*/
|
|
dropExecutables: boolean
|
|
/**
|
|
* Posters whose uploads are dropped whatever they are: fragments of the poster lowercased without punctuation
|
|
* ("@__Celeb_Movie__@" → "celebmovie"). The CelebMovie family's scraped celebrity pictures, the same files again and
|
|
* again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)).
|
|
*/
|
|
droppedPosters: string[]
|
|
}
|
|
|
|
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
|
version: 1,
|
|
droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'],
|
|
perArticleMaxParts: 3,
|
|
samplePercent: 1,
|
|
dropExecutables: true,
|
|
droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
|
|
}
|
|
|
|
export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
|
|
|
|
/** A poster as droppedPosters matches it: lowercased, letters and digits only. */
|
|
export function posterWords(poster: string): string {
|
|
return poster.replace(/[^A-Za-z0-9]/g, '').toLowerCase()
|
|
}
|
|
|
|
/** Files that come along with a release's content: they don't make it one of executables or not. */
|
|
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
|
|
const EXECUTABLE = /\.(exe|scr|com|pif|bat|cmd|vbs|lnk)\s*$/i
|
|
|
|
export function isRandomFile(name: string) {
|
|
return looksRandom(fileStem(name))
|
|
}
|
|
|
|
/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
|
|
export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
|
|
let owner: string | null = null
|
|
for (const g of groups) {
|
|
const p = priorities[g]
|
|
if (p !== undefined && (owner === null || p < priorities[owner])) owner = g
|
|
}
|
|
return owner
|
|
}
|
|
|
|
export class RuleSet {
|
|
private signatures: Set<string>
|
|
private tools: Set<string>
|
|
private posters: string[]
|
|
constructor(readonly rules: Rules) {
|
|
this.posters = rules.droppedPosters ?? []
|
|
this.signatures = new Set(rules.droppedSignatures)
|
|
this.tools = new Set(rules.droppedTools)
|
|
}
|
|
|
|
/**
|
|
* A release's verdict from its files: dropped when its poster is; when its content is bare
|
|
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
|
|
* dropped, as per-article random subjects when every file has few parts.
|
|
*/
|
|
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict {
|
|
const sample = explored(releaseKey, this.rules.samplePercent)
|
|
if (this.posters.length > 0 && files.length > 0) {
|
|
const words = posterWords(files[0].poster)
|
|
const dropped = this.posters.find((p) => words.includes(p))
|
|
if (dropped) return { rule: 'poster', key: dropped, sample }
|
|
}
|
|
if (this.rules.dropExecutables) {
|
|
const content = files.filter((f) => !SIDECAR.test(f.name))
|
|
if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
|
|
return { rule: 'executable', key: EXECUTABLE.exec(content[0].name)![1].toLowerCase(), sample }
|
|
}
|
|
}
|
|
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
|
|
const first = files[0]
|
|
const tool = uploadTool(first.firstMessageId)
|
|
// Garbage first: a dropped tool or signature isn't worth reading bodies for either.
|
|
const group = first.groups[0] ?? ''
|
|
if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample }
|
|
if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample }
|
|
return { rule: null }
|
|
}
|
|
}
|