From 3d24db55653dddf44f73e288f8087898d2806996 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 16 Oct 2026 11:36:46 +0200 Subject: [PATCH] Rules drop posters at the header pass: the CelebMovie family (rule set version 3) Co-Authored-By: Claude Opus 5.5 --- app/services/fragment_merger.ts | 2 +- .../1770000003800_rules_dropped_posters.ts | 24 +++++++++++ edge/src/bin/edge-node.rs | 5 +++ edge/src/indexer.rs | 2 +- edge/src/names.rs | 42 +++++++++++++++++++ lib/edge/range_indexer.ts | 4 +- lib/rules/index.ts | 2 +- lib/rules/rule_set.ts | 25 +++++++++-- tests/unit/rules/rule_set.spec.ts | 19 ++++++++- 9 files changed, 116 insertions(+), 9 deletions(-) create mode 100644 database/migrations/1770000003800_rules_dropped_posters.ts diff --git a/app/services/fragment_merger.ts b/app/services/fragment_merger.ts index df708fc..0d86cd9 100644 --- a/app/services/fragment_merger.ts +++ b/app/services/fragment_merger.ts @@ -83,7 +83,7 @@ class FragmentMerger { } const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = [] for (const [key, { files, rangeId }] of byRelease) { - const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId }))) + const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster }))) if (verdict.rule !== null) continue uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) })) } diff --git a/database/migrations/1770000003800_rules_dropped_posters.ts b/database/migrations/1770000003800_rules_dropped_posters.ts new file mode 100644 index 0000000..947d3c9 --- /dev/null +++ b/database/migrations/1770000003800_rules_dropped_posters.ts @@ -0,0 +1,24 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * A rule set version with droppedPosters (lib/rules/rule_set.ts): the newest set plus the CelebMovie family, whose + * scraped celebrity pictures workers drop at the header pass from now on (LOG.md 2026-10-16 (65)). A new version, so the + * ranges record which ran with it. Nothing where there's no rule set yet (the default gets it) or the newest has it. + */ +const POSTERS = ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'] + +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + const latest = await db.from('rule_sets').orderBy('version', 'desc').first() + if (!latest || 'droppedPosters' in latest.rules) return + await db.table('rule_sets').insert({ + version: Number(latest.version) + 1, + rules: JSON.stringify({ ...latest.rules, droppedPosters: POSTERS }), + note: 'dropped posters: the CelebMovie family (2026-10-16)', + }) + }) + } + + async down() {} +} diff --git a/edge/src/bin/edge-node.rs b/edge/src/bin/edge-node.rs index 915ac7a..0e2f134 100644 --- a/edge/src/bin/edge-node.rs +++ b/edge/src/bin/edge-node.rs @@ -64,6 +64,8 @@ struct RulesJson { sample_percent: Option, #[serde(default)] drop_executables: Option, + #[serde(default)] + dropped_posters: Option>, } impl RulesJson { @@ -83,6 +85,9 @@ impl RulesJson { if let Some(e) = self.drop_executables { rules.drop_executables = e; } + if let Some(posters) = self.dropped_posters { + rules.dropped_posters = posters; + } rules } } diff --git a/edge/src/indexer.rs b/edge/src/indexer.rs index e83eb1e..6f75ecf 100644 --- a/edge/src/indexer.rs +++ b/edge/src/indexer.rs @@ -752,7 +752,7 @@ impl RangeIndexer { for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() { let judged: Vec = list .iter() - .map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id }) + .map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster }) .collect(); match names::judge(&key, &judged, &self.rules) { Verdict::Keep => { diff --git a/edge/src/names.rs b/edge/src/names.rs index bacb8b5..e409f2a 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -1241,6 +1241,7 @@ pub struct JudgedFile<'a> { pub total_parts: u32, pub groups: &'a [String], pub first_message_id: &'a str, + pub poster: &'a str, } pub enum Verdict { @@ -1260,6 +1261,8 @@ pub struct Rules { pub per_article_max_parts: u32, pub sample_percent: f64, pub drop_executables: bool, + /// Fragments of the poster lowercased without punctuation (droppedPosters). + pub dropped_posters: Vec, } impl Rules { @@ -1273,13 +1276,25 @@ impl Rules { per_article_max_parts: 3, sample_percent: 1.0, drop_executables: true, + dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(), } } } +/// posterWords: lowercased, letters and digits only. +pub fn poster_words(poster: &str) -> String { + poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect() +} + /// RuleSet.judge. pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict { let sample = explored(release_key, rules.sample_percent); + if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) { + let words = poster_words(first.poster); + if let Some(dropped) = rules.dropped_posters.iter().find(|p| words.contains(p.as_str())) { + return Verdict::Drop { rule: "poster", key: dropped.clone(), sample }; + } + } let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect(); if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) { let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap(); @@ -1459,6 +1474,33 @@ mod numbered_tests { } } +#[cfg(test)] +mod poster_rule_tests { + use super::{judge, JudgedFile, Rules, Verdict}; + + fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> { + let groups = vec!["alt.binaries.erotica".to_string()]; + let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "", poster }]; + match judge("k", &files, rules) { + Verdict::Keep => None, + Verdict::Drop { rule, key, .. } => Some((rule, key)), + } + } + + #[test] + fn a_dropped_posters_uploads_are_dropped() { + let rules = Rules::defaults(Default::default()); + assert_eq!(verdict("@_CelebMovie_@", &rules), Some(("poster", "celebmovie".to_string()))); + assert_eq!(verdict("@__Celeb_Movie__@", &rules), Some(("poster", "celebmovie".to_string()))); + assert_eq!(verdict("NudeCelebs@CelebWow.com", &rules), Some(("poster", "celebwow".to_string()))); + assert_eq!(verdict("@nude-celebs", &rules), Some(("poster", "nudecelebs".to_string()))); + assert_eq!(verdict("celeborn@not.here (Celeborn)", &rules), None); + assert_eq!(verdict(r#""CelebTA.com" "#, &rules), None); + let none = Rules { dropped_posters: vec![], ..Rules::defaults(Default::default()) }; + assert_eq!(verdict("@_CelebMovie_@", &none), None); + } +} + #[cfg(test)] mod image_set_tests { use super::{release_identity, sized_name, subject_remainder, subject_remainder_re}; diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 7934fde..9141d68 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -48,7 +48,7 @@ export interface ProbeCandidate { /** What a rule dropped in a batch (REBUILD.md, rules and recovery). */ export interface RuleCount { - rule: 'crosspost' | 'obfuscated' | 'executable' | 'garbage' | 'per-article' | 'incomplete' + rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'incomplete' key: string articles: number files: number @@ -215,7 +215,7 @@ export class RangeIndexer { uploadsOf(all).forEach((list, upload) => { const verdict = this.rules.judge( key, - list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId })) + list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })) ) if (verdict.rule === null) { this.batch.releases.push(chunkOf(key, list, upload)) diff --git a/lib/rules/index.ts b/lib/rules/index.ts index 112f217..f917b64 100644 --- a/lib/rules/index.ts +++ b/lib/rules/index.ts @@ -1,2 +1,2 @@ export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js' -export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, type Rules, type Verdict } from './rule_set.js' +export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js' diff --git a/lib/rules/rule_set.ts b/lib/rules/rule_set.ts index ed8edf0..09b5f2f 100644 --- a/lib/rules/rule_set.ts +++ b/lib/rules/rule_set.ts @@ -24,6 +24,12 @@ export interface Rules { * 2026-10-09). Genuine software comes in archives. */ dropExecutables: boolean + /** + * Posters whose uploads are dropped whatever they are: fragments of the poster lowercased without punctuation + * ("@__Celeb_Movie__@" → "celebmovie"). The CelebMovie family's scraped celebrity pictures, the same files again and + * again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)). + */ + droppedPosters: string[] } export const DEFAULT_RULES: Omit = { @@ -32,9 +38,15 @@ export const DEFAULT_RULES: Omit = { perArticleMaxParts: 3, samplePercent: 1, dropExecutables: true, + droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'], } -export type Verdict = { rule: null } | { rule: 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean } +export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean } + +/** A poster as droppedPosters matches it: lowercased, letters and digits only. */ +export function posterWords(poster: string): string { + return poster.replace(/[^A-Za-z0-9]/g, '').toLowerCase() +} /** Files that come along with a release's content: they don't make it one of executables or not. */ const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i @@ -57,18 +69,25 @@ export function ownerOf(groups: string[], priorities: Record): s export class RuleSet { private signatures: Set private tools: Set + private posters: string[] constructor(readonly rules: Rules) { + this.posters = rules.droppedPosters ?? [] this.signatures = new Set(rules.droppedSignatures) this.tools = new Set(rules.droppedTools) } /** - * A release's verdict from its files: dropped when its content is bare + * A release's verdict from its files: dropped when its poster is; when its content is bare * executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is * dropped, as per-article random subjects when every file has few parts. */ - judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict { + judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict { const sample = explored(releaseKey, this.rules.samplePercent) + if (this.posters.length > 0 && files.length > 0) { + const words = posterWords(files[0].poster) + const dropped = this.posters.find((p) => words.includes(p)) + if (dropped) return { rule: 'poster', key: dropped, sample } + } if (this.rules.dropExecutables) { const content = files.filter((f) => !SIDECAR.test(f.name)) if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) { diff --git a/tests/unit/rules/rule_set.spec.ts b/tests/unit/rules/rule_set.spec.ts index 20c6fd8..427988f 100644 --- a/tests/unit/rules/rule_set.spec.ts +++ b/tests/unit/rules/rule_set.spec.ts @@ -3,7 +3,13 @@ import { DEFAULT_RULES, RuleSet, ownerOf, uploadTool, explored } from '#lib/rule import { sevenZipVolume } from '#lib/files/index' const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] }) -const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath') => ({ name, totalParts, groups: [group], firstMessageId }) +const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath', poster = 'someone@example.com (Someone)') => ({ + name, + totalParts, + groups: [group], + firstMessageId, + poster, +}) const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost' const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu' @@ -17,6 +23,17 @@ test.group('rules / owner', () => { }) test.group('rules / verdicts', () => { + test("a dropped poster's uploads are dropped whatever they are; others with the word in their name are not", ({ assert }) => { + const pics = (poster: string) => rules.judge('k', [file('!index0001.jpg', 1, NYUU, 'alt.binaries.erotica', poster)]) + assert.include(pics('@_CelebMovie_@'), { rule: 'poster', key: 'celebmovie' }) + assert.include(pics('@__Celeb_Movie__@'), { rule: 'poster', key: 'celebmovie' }) + assert.include(pics('NudeCelebs@CelebWow.com'), { rule: 'poster', key: 'celebwow' }) + assert.include(pics('@nude-celebs'), { rule: 'poster', key: 'nudecelebs' }) + assert.deepEqual(pics('celeborn@not.here (Celeborn)'), { rule: null }) + assert.deepEqual(pics('"CelebTA.com" '), { rule: null }) + assert.deepEqual(new RuleSet({ ...DEFAULT_RULES, droppedPosters: [], samplePercent: 0, priorities: {}, droppedSignatures: [] }).judge('k', [file('a.jpg', 1, NYUU, 'g', '@_CelebMovie_@')]), { rule: null }) + }) + test('a readable file keeps its whole release', ({ assert }) => { assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null }) })