From e470e231d0857e1dca9e04da16c4dac2c88922e0 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sat, 31 Oct 2026 11:35:46 +0100 Subject: [PATCH] Single-archive rules: stub archives and nonsense pieces dropped at the edge The judge gets each file's articles seen and bytes, and whether the window closed on the release. A settled release of one unsplit archive posted to 2+ groups is dropped as a stub archive when its articles are <= 256 KiB with the release >= 1 MB, or it is in 4+ groups, or by a webmail poster (those two under 100 MB); releases touching music, e-book, comics, tabs or fonts groups are exempt. A one-article archive of 100 kB-1 MB under a release-like, unscrambled name without spaces or an e-book format is dropped as a nonsense piece. Both switched by the rule set (dropStubArchives, dropNonsensePieces), mirrored in lib/rules, sampled like every drop (LOG.md (123)-(127)). Co-Authored-By: Claude Fable 5.1 --- REBUILD.md | 36 ++++++- edge/src/indexer.rs | 16 ++- edge/src/lease.rs | 10 ++ edge/src/names.rs | 172 +++++++++++++++++++++++++++++- lib/edge/range_indexer.ts | 15 ++- lib/rules/index.ts | 2 +- lib/rules/rule_set.ts | 107 ++++++++++++++++++- tests/unit/rules/rule_set.spec.ts | 54 ++++++++++ 8 files changed, 397 insertions(+), 15 deletions(-) diff --git a/REBUILD.md b/REBUILD.md index 35cd79d..022f4bf 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -151,7 +151,9 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal: indexed groups in the article's Xref, the one with the lowest priority number; see the group list) → group articles into files → files into releases (`lib/release` rules) → drop garbage (random name + - dropped signature or tool) → set aside per-article random subjects and + dropped signature or tool), bare executables, dropped posters, and the + single-archive fakes (stub archives and nonsense pieces, below) → set + aside per-article random subjects and obfuscated subjects (counted only; phase 2). Returns per batch: release chunks, edge fragments, probe candidates, counters. Holds no state beyond its lease. @@ -776,7 +778,37 @@ its release and sets `partial` (the store: `complete = false`). A lone article of a multi-part random file is still counted `per-article`, not written; before the block's end a scrambled name counts as random too (`looks_scrambled`: a random subject and poster per article, 160k of -alt.binaries.storage's releases; LOG.md (122)). Gone: fragments and the cut-off, `uploads_of` (online now), +alt.binaries.storage's releases; LOG.md (122)). + +**Single-archive rules (built 2026-10-31; LOG.md (123)-(127)).** The judge +(`names::judge`, `RuleSet.judge`) gets each file's articles seen and their +bytes, and whether the release is *settled* (the window closed on it: a +par2 can still join an open one; the start edge doesn't count, since a par2 +follows its rar and 7% of the catalog's releases are start edges). A settled +release of one unsplit zip, rar or 7z and nothing else is dropped as a +**stub archive** (`stub-archive`, key: the legs tripped) when posted to +2+ groups and (A) its articles are ≤ 256 KiB with the release ≥ 1 MB +(Power-Post's 250 kB default; a one-article release always has small +articles, which exempts e-books and sidecars), or (X) posted to 4+ groups, +or (W) by a webmail poster, X and W under 100 MB (the fakes stop at +~90 MB; hash-named uploads and real albums above it don't). A release +touching a music, e-book, comics, tabs or fonts group is exempt: those +communities post real single small archives the same way (karaoke zips to +a.b.sounds.karaoke + a.b.test with Power-Post's articles; 16 of 32,308 +known fakes lost, LOG.md (127)). These are the +exe-stub and recycled-installer floods under film and software names: +55% of the catalog on 2026-10-30, 99.3% of the known fakes, 0 known +content in the 217k sample, verified on 1,500 decoded articles. A settled +single archive of one article and 100 kB–1 MB under a release-like, +unscrambled name without spaces and without an e-book format in it is +dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool): +the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of +the catalog; under 100 kB are cue/log sidecars and adverts, and the names +with spaces are books and albums. Both rules are switched by the +rule set (`dropStubArchives`, `dropNonsensePieces`), sampled like every +drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer +judges as each batch completes files, so it passes settled without the +window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now), `check.ts` and the fixture comparison (the parity with the TypeScript indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count (incomplete releases are kept). The wire carries `finished`, `edge` and diff --git a/edge/src/indexer.rs b/edge/src/indexer.rs index f50a783..5c54515 100644 --- a/edge/src/indexer.rs +++ b/edge/src/indexer.rs @@ -1423,9 +1423,21 @@ impl RangeIndexer { fn write(&mut self, key: &str, upload: usize, list: Vec>, finished: bool, edge: bool, album: Option) { let judged: Vec = list .iter() - .map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster }) + .map(|f| JudgedFile { + name: &f[0].file.name, + total_parts: f[0].file.total_parts, + groups: &f[0].file.groups, + first_message_id: &f[0].message_id, + poster: &f[0].file.poster, + segments: f.len() as u32, + bytes: f.iter().map(|s| s.bytes.max(0) as u64).sum(), + }) .collect(); - match names::judge(key, &judged, &self.rules) { + // Settled for the single-archive rules: the window closed on it (a par2 can still join an open one). The + // start edge (posted within the window of the block's start) doesn't count: a par2 comes after its rar, so + // one in the previous block is rare (0 par2 siblings of 3,000 catalog hits, LOG.md (125)), and 7% of the + // catalog's releases are start edges, which would escape the rule. + match names::judge(key, &judged, finished, &self.rules) { Verdict::Keep => { let candidates = candidates_of(key, &list, upload); let mut chunk = chunk_of(key, &list, upload); diff --git a/edge/src/lease.rs b/edge/src/lease.rs index 12d7399..f5c2ef7 100644 --- a/edge/src/lease.rs +++ b/edge/src/lease.rs @@ -71,6 +71,10 @@ pub struct RulesJson { pub drop_executables: Option, #[serde(default)] pub dropped_posters: Option>, + #[serde(default)] + pub drop_stub_archives: Option, + #[serde(default)] + pub drop_nonsense_pieces: Option, } impl RulesJson { @@ -94,6 +98,12 @@ impl RulesJson { if let Some(posters) = self.dropped_posters { rules.dropped_posters = posters; } + if let Some(d) = self.drop_stub_archives { + rules.drop_stub_archives = d; + } + if let Some(d) = self.drop_nonsense_pieces { + rules.drop_nonsense_pieces = d; + } rules } } diff --git a/edge/src/names.rs b/edge/src/names.rs index bd95e48..a82db15 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -1296,6 +1296,31 @@ pub fn is_sidecar(name: &str) -> bool { test(&SIDECAR, name) } static EXECUTABLE: LazyLock = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$")); +/// An unsplit archive (the judge's ARCHIVE and VOLUME): one zip, rar or 7z that is not a volume of a set. +static ARCHIVE: LazyLock = LazyLock::new(|| re(r"(?i)\.(zip|rar|7z)$")); +static VOLUME: LazyLock = LazyLock::new(|| re(r"(?i)\.(part[0-9]+\.rar|r[0-9][0-9]|z[0-9][0-9]|7z\.[0-9][0-9][0-9]|[0-9][0-9][0-9])$")); +/// A poster at a free mail provider (the judge's WEBMAIL): the stub floods' throwaway accounts. +static WEBMAIL: LazyLock = LazyLock::new(|| { + re(r"(?i)@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\.") +}); +/// A name that reads as a release (the judge's RELEASE_LIKE): a year, a scene tag, a group suffix or dotted words. +static RELEASE_LIKE: LazyLock = LazyLock::new(|| { + re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)") +}); + +/// The stub-archive rule (LOG.md (123)-(126)): one unsplit archive, no par2, settled, posted to 2+ groups, and its +/// articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's 250 kB default; a +/// one-article release always has small articles, so e-books and sidecars are exempt), or 4+ groups, or a webmail +/// poster, these two under STUB_MAX_BYTES (no article-size signal: the fakes stop at ~90 MB, the hash-named +/// uploads and real albums above it don't). +pub const STUB_ARTICLE_BYTES: u64 = 262_144; +pub const STUB_MIN_BYTES: u64 = 1_000_000; +pub const STUB_MAX_BYTES: u64 = 100_000_000; +pub const STUB_WIDE_GROUPS: usize = 4; +/// The nonsense-piece rule: one article holding one archive of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a +/// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts). +pub const NONSENSE_MIN_BYTES: u64 = 100_000; +pub const NONSENSE_MAX_BYTES: u64 = 1_000_000; pub struct JudgedFile<'a> { pub name: &'a str, @@ -1303,6 +1328,58 @@ pub struct JudgedFile<'a> { pub groups: &'a [String], pub first_message_id: &'a str, pub poster: &'a str, + /// Articles seen and their bytes (the overview's :bytes). + pub segments: u32, + pub bytes: u64, +} + +/// One unsplit archive and nothing else. +fn single_archive<'a, 'b>(files: &'b [JudgedFile<'a>]) -> Option<&'b JudgedFile<'a>> { + match files { + [f] if test(&ARCHIVE, f.name) && !test(&VOLUME, f.name) => Some(f), + _ => None, + } +} + +/// Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way +/// the stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the +/// stub-archive rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)). +static STUB_EXEMPT_GROUPS: LazyLock = LazyLock::new(|| re(r"(?i)(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)")); + +/// The legs of the stub-archive rule a settled single archive trips ("A" article size, "X" 4+ groups, "W" webmail), +/// empty when none. +pub fn stub_archive_legs(f: &JudgedFile) -> String { + let mut legs = String::new(); + if f.groups.len() < 2 || f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) { + return legs; + } + let article = f.bytes / u64::from(f.segments.max(1)); + if article <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES { + legs.push('A'); + } + if f.groups.len() >= STUB_WIDE_GROUPS && f.bytes < STUB_MAX_BYTES { + legs.push('X'); + } + if f.bytes < STUB_MAX_BYTES && test(&WEBMAIL, f.poster) { + legs.push('W'); + } + legs +} + +/// An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece. +static EBOOK: LazyLock = LazyLock::new(|| re(r"(?i)\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b")); + +/// A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the +/// newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces). +pub fn is_nonsense_piece(f: &JudgedFile) -> bool { + f.total_parts == 1 + && f.segments == 1 + && (NONSENSE_MIN_BYTES..NONSENSE_MAX_BYTES).contains(&f.bytes) + && !f.name.contains(' ') + && test(&RELEASE_LIKE, &file_stem(f.name)) + && !test(&EBOOK, f.name) + && !looks_scrambled(f.name) + && !is_random_file(f.name) } pub enum Verdict { @@ -1324,6 +1401,9 @@ pub struct Rules { pub drop_executables: bool, /// Fragments of the poster lowercased without punctuation (droppedPosters). pub dropped_posters: Vec, + /// The stub-archive rule (dropStubArchives) and the nonsense-piece rule (dropNonsensePieces). + pub drop_stub_archives: bool, + pub drop_nonsense_pieces: bool, } impl Rules { @@ -1338,6 +1418,8 @@ impl Rules { sample_percent: 1.0, drop_executables: true, dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(), + drop_stub_archives: true, + drop_nonsense_pieces: true, } } } @@ -1347,8 +1429,9 @@ pub fn poster_words(poster: &str) -> String { poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect() } -/// RuleSet.judge. -pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict { +/// RuleSet.judge. `settled`: the window closed on the release (a par2 can still join an open one), which the +/// single-archive rules need. +pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rules) -> Verdict { let sample = explored(release_key, rules.sample_percent); if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) { let words = poster_words(first.poster); @@ -1362,6 +1445,17 @@ pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase(); return Verdict::Drop { rule: "executable", key, sample }; } + if let (true, Some(f)) = (settled, single_archive(files)) { + if rules.drop_stub_archives { + let legs = stub_archive_legs(f); + if !legs.is_empty() { + return Verdict::Drop { rule: "stub-archive", key: legs, sample }; + } + } + if rules.drop_nonsense_pieces && is_nonsense_piece(f) { + return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample }; + } + } if files.iter().any(|f| !is_random_file(f.name)) { return Verdict::Keep; } @@ -1567,8 +1661,8 @@ mod poster_rule_tests { fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> { let groups = vec!["alt.binaries.erotica".to_string()]; - let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "", poster }]; - match judge("k", &files, rules) { + let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "", poster, segments: 1, bytes: 200_000 }]; + match judge("k", &files, true, rules) { Verdict::Keep => None, Verdict::Drop { rule, key, .. } => Some((rule, key)), } @@ -1588,6 +1682,76 @@ mod poster_rule_tests { } } +#[cfg(test)] +mod single_archive_rule_tests { + use super::{judge, JudgedFile, Rules, Verdict}; + + const NYUU: &str = "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"; + + fn verdict(name: &str, poster: &str, groups: usize, segments: u32, bytes: u64, settled: bool, rules: &Rules) -> Option<(&'static str, String)> { + let groups: Vec = (0..groups).map(|i| format!("alt.binaries.g{i}")).collect(); + let files = [JudgedFile { name, total_parts: segments, groups: &groups, first_message_id: NYUU, poster, segments, bytes }]; + match judge("k", &files, settled, rules) { + Verdict::Keep => None, + Verdict::Drop { rule, key, .. } => Some((rule, key)), + } + } + + #[test] + fn stub_archives_by_article_size_groups_or_webmail() { + let rules = Rules::defaults(Default::default()); + // Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave. + assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com (Jahnunsinos007)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "AW".to_string()))); + assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "Yenc@power-post.org (Yenc-PP-A&A)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "A".to_string()))); + // Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone). + assert_eq!(verdict("Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar", "Yenc@power-post.org (Yenc-PP-A&A)", 5, 2, 319_000, true, &rules), Some(("stub-archive", "X".to_string()))); + // A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave. + assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 400, 260_000_000, true, &rules), None, "over 100 MB without the article-size leg"); + assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 100, 65_000_000, true, &rules), Some(("stub-archive", "W".to_string()))); + // Settled only: a par2 can still join an open release. + assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, false, &rules), None); + } + + #[test] + fn small_and_single_group_and_split_archives_are_kept() { + let rules = Rules::defaults(Default::default()); + // An e-book: one or two small articles always have small articles (LOG.md (126)). + assert_eq!(verdict("Maeve Binchy - Tara Road (epub).rar", "kie@home.now (KiE)", 2, 2, 521_000, true, &rules), None); + // One group: most real single rars. + assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 1, 48, 12_000_000, true, &rules), None); + // Modern 700 kB articles, no webmail, two groups. + assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "poster@example.org", 2, 100, 70_000_000, true, &rules), None); + // A volume, not a single archive; a hash-named 2 GB upload from a gmail-looking poster. + assert_eq!(verdict("Some.Movie.2014.part01.rar", "a@gmail.com", 2, 48, 12_000_000, true, &rules), None); + assert_eq!(verdict("cbcd32359460df1fa9b7d01d8720.rar", "b648de0d1c0f7a9a77be273d@gmail.com", 2, 3163, 2_340_000_000, true, &rules), None); + let off = Rules { drop_stub_archives: false, ..Rules::defaults(Default::default()) }; + assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, true, &off), None); + // A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)). + let groups = vec!["alt.binaries.sounds.karaoke".to_string(), "alt.binaries.test".to_string()]; + let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl.zip", total_parts: 13, groups: &groups, first_message_id: NYUU, poster: "home@home.com (TexasDj)", segments: 13, bytes: 3_300_000 }]; + assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep)); + } + + #[test] + fn nonsense_pieces() { + let rules = Rules::defaults(Default::default()); + // One 790 kB article under a release-like name: the newznzb.* pieces. + assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string()))); + // A 3 kB cue/log sidecar rar with a year in its name, and a diet advert: under the floor. + assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), None); + // A book (a year or a format in its name, written with spaces), a dotted one too: not a piece. + assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None); + assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None); + assert_eq!(verdict("Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar", "saggir@domain.com", 1, 1, 113_000, true, &rules), None); + // Two articles, or no release-like name, or scrambled: not this rule. + assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 2, 790_000, true, &rules), None); + assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None); + assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None); + let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) }; + assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None); + } +} + #[cfg(test)] mod image_set_tests { use super::{release_identity, sized_name, subject_remainder, subject_remainder_re}; diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 9f5add3..607721f 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -62,7 +62,7 @@ export interface ProbeCandidate { /** What a rule dropped in a batch (REBUILD.md, rules and recovery). */ export interface RuleCount { - rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'incomplete' + rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'incomplete' key: string articles: number files: number @@ -246,9 +246,20 @@ export class RangeIndexer { this.completed = new Map() for (const [key, all] of byRelease) { uploadsOf(all).forEach((list, upload) => { + // Settled as far as this indexer knows: it judges a release's files as each batch completes them (the Rust + // window passes whether the window closed on the release and no block edge cut it). const verdict = this.rules.judge( key, - list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })) + list.map((f) => ({ + name: f.segments[0].file.name, + totalParts: f.segments[0].totalParts, + groups: f.segments[0].file.groups, + firstMessageId: f.segments[0].messageId, + poster: f.segments[0].file.poster, + segments: f.segments.length, + bytes: f.segments.reduce((n, s) => n + s.bytes, 0), + })), + true ) if (verdict.rule === null) { this.batch.releases.push(chunkOf(key, list, upload)) diff --git a/lib/rules/index.ts b/lib/rules/index.ts index 0906a78..a292266 100644 --- a/lib/rules/index.ts +++ b/lib/rules/index.ts @@ -1,2 +1,2 @@ export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js' -export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js' +export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, stubArchiveLegs, isNonsensePiece, STUB_ARTICLE_BYTES, STUB_MIN_BYTES, STUB_MAX_BYTES, STUB_WIDE_GROUPS, NONSENSE_MIN_BYTES, NONSENSE_MAX_BYTES, type Rules, type Verdict, type JudgedFile } from './rule_set.js' diff --git a/lib/rules/rule_set.ts b/lib/rules/rule_set.ts index 65cf6cb..df25476 100644 --- a/lib/rules/rule_set.ts +++ b/lib/rules/rule_set.ts @@ -30,6 +30,18 @@ export interface Rules { * again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)). */ droppedPosters: string[] + /** + * Drop settled releases of one unsplit archive without a par2, posted to 2+ groups, whose articles are at most + * STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES, or posted to 4+ groups, or by a webmail poster, the + * last two under STUB_MAX_BYTES: the exe-stub and recycled-installer floods under film and software names (55% + * of the catalog on 2026-10-30; 99.3% of the known fakes, 0 known content; LOG.md (123)-(126)). + */ + dropStubArchives: boolean + /** + * Drop one-article archives of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a release-like, unscrambled name: + * the newznzb.* pieces (9% of the catalog; LOG.md (123), (125)). + */ + dropNonsensePieces: boolean } export const DEFAULT_RULES: Omit = { @@ -39,9 +51,31 @@ export const DEFAULT_RULES: Omit = { samplePercent: 1, dropExecutables: true, droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'], + dropStubArchives: true, + dropNonsensePieces: true, } -export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean } +/** The stub-archive rule's bounds (edge/src/names.rs): Power-Post's 250 kB default article, the fakes' size span. */ +export const STUB_ARTICLE_BYTES = 262144 +export const STUB_MIN_BYTES = 1_000_000 +export const STUB_MAX_BYTES = 100_000_000 +export const STUB_WIDE_GROUPS = 4 +/** The nonsense-piece rule's bounds: under 100 kB are cue/log sidecars and adverts, 1 MB and up is rule 1's. */ +export const NONSENSE_MIN_BYTES = 100_000 +export const NONSENSE_MAX_BYTES = 1_000_000 + +export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece'; key: string; sample: boolean } + +/** A file as the judge sees it: its name, parts, groups, poster, and the articles seen with their bytes. */ +export interface JudgedFile { + name: string + totalParts: number + groups: string[] + firstMessageId: string + poster: string + segments: number + bytes: number +} /** A poster as droppedPosters matches it: lowercased, letters and digits only. */ export function posterWords(poster: string): string { @@ -51,6 +85,61 @@ export function posterWords(poster: string): string { /** Files that come along with a release's content: they don't make it one of executables or not. */ const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i const EXECUTABLE = /\.(exe|scr|com|pif|bat|cmd|vbs|lnk)\s*$/i +/** One zip, rar or 7z that is not a volume of a set. */ +const ARCHIVE = /\.(zip|rar|7z)$/i +const VOLUME = /\.(part\d+\.rar|r\d\d|z\d\d|7z\.\d\d\d|\d\d\d)$/i +/** A poster at a free mail provider: the stub floods' throwaway accounts. */ +const WEBMAIL = /@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\./i +/** A name that reads as a release: a year, a scene tag, a group suffix or dotted words. */ +const RELEASE_LIKE = /(\b(19|20)\d\d\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s\d\de\d\d|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v\d+(\.\d+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)/i + +/** + * Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way the + * stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the stub-archive + * rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)). + */ +const STUB_EXEMPT_GROUPS = /(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)/i + +/** The single unsplit archive a release consists of, or null. */ +function singleArchive(files: JudgedFile[]): JudgedFile | null { + return files.length === 1 && ARCHIVE.test(files[0].name) && !VOLUME.test(files[0].name) ? files[0] : null +} + +/** + * The legs of the stub-archive rule a settled single archive trips: "A" (articles <= STUB_ARTICLE_BYTES, release >= + * STUB_MIN_BYTES: a one-article release always has small articles, so e-books and sidecars are exempt), "X" (4+ + * groups) and "W" (webmail poster), the last two under STUB_MAX_BYTES (the fakes stop at ~90 MB; hash-named uploads + * and real albums above it don't). Empty when none. + */ +export function stubArchiveLegs(f: JudgedFile): string { + if (f.groups.length < 2 || f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g))) return '' + let legs = '' + if (f.bytes / Math.max(f.segments, 1) <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES) legs += 'A' + if (f.groups.length >= STUB_WIDE_GROUPS && f.bytes < STUB_MAX_BYTES) legs += 'X' + if (f.bytes < STUB_MAX_BYTES && WEBMAIL.test(f.poster)) legs += 'W' + return legs +} + +/** An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece. */ +const EBOOK = /\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b/i + +/** + * A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the + * newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces). + */ +export function isNonsensePiece(f: JudgedFile): boolean { + return ( + f.totalParts === 1 && + f.segments === 1 && + f.bytes >= NONSENSE_MIN_BYTES && + f.bytes < NONSENSE_MAX_BYTES && + !f.name.includes(' ') && + RELEASE_LIKE.test(fileStem(f.name)) && + !EBOOK.test(f.name) && + !isScrambledFile(f.name) && + !isRandomFile(f.name) + ) +} export function isRandomFile(name: string) { return looksRandom(fileStem(name)) @@ -98,10 +187,12 @@ export class RuleSet { /** * A release's verdict from its files: dropped when its poster is; when its content is bare - * executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is - * dropped, as per-article random subjects when every file has few parts. + * executables; when it is a settled single archive that is a stub or a nonsense piece; else kept as a whole when + * any file has a readable name; else dropped as garbage when its signature or tool is dropped, as per-article + * random subjects when every file has few parts. `settled`: the window closed on the release (a par2 can still + * join an open one), which the single-archive rules need. */ - judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict { + judge(releaseKey: string, files: JudgedFile[], settled = true): Verdict { const sample = explored(releaseKey, this.rules.samplePercent) if (this.posters.length > 0 && files.length > 0) { const words = posterWords(files[0].poster) @@ -114,6 +205,14 @@ export class RuleSet { return { rule: 'executable', key: EXECUTABLE.exec(content[0].name)![1].toLowerCase(), sample } } } + const archive = settled ? singleArchive(files) : null + if (archive) { + if (this.rules.dropStubArchives) { + const legs = stubArchiveLegs(archive) + if (legs !== '') return { rule: 'stub-archive', key: legs, sample } + } + if (this.rules.dropNonsensePieces && isNonsensePiece(archive)) return { rule: 'nonsense-piece', key: uploadTool(archive.firstMessageId), sample } + } if (files.some((f) => !isRandomFile(f.name))) return { rule: null } const first = files[0] const tool = uploadTool(first.firstMessageId) diff --git a/tests/unit/rules/rule_set.spec.ts b/tests/unit/rules/rule_set.spec.ts index 427988f..fcb7a8b 100644 --- a/tests/unit/rules/rule_set.spec.ts +++ b/tests/unit/rules/rule_set.spec.ts @@ -9,6 +9,18 @@ const file = (name: string, totalParts: number, firstMessageId: string, group = groups: [group], firstMessageId, poster, + segments: totalParts, + bytes: totalParts * 700_000, +}) +/** A single archive of `segments` articles and `bytes` in all, posted to `groups` groups. */ +const archive = (name: string, poster: string, groups: number, segments: number, bytes: number, firstMessageId = NYUU) => ({ + name, + totalParts: segments, + groups: Array.from({ length: groups }, (_, i) => `alt.binaries.g${i}`), + firstMessageId, + poster, + segments, + bytes, }) const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost' const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu' @@ -50,6 +62,48 @@ test.group('rules / verdicts', () => { assert.include(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU, 'alt.binaries.misc')]), { rule: 'garbage', key: 'nyuu alt.binaries.misc' }) assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NYUU)]), { rule: null }) }) + + test('stub archives: a settled single archive in 2+ groups with 250 kB articles, or 4+ groups, or a webmail poster', ({ assert }) => { + // Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave; the article-size leg alone without webmail. + assert.include(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com (Jahnunsinos007)', 2, 48, 12_000_000)]), { rule: 'stub-archive', key: 'AW' }) + assert.include(rules.judge('k', [archive('Some.Movie.2014.720p.BluRay-GRP.rar', 'Yenc@power-post.org (Yenc-PP-A&A)', 2, 48, 12_000_000)]), { rule: 'stub-archive', key: 'A' }) + // Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone). + assert.include(rules.judge('k', [archive('Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar', 'Yenc@power-post.org', 5, 2, 319_000)]), { rule: 'stub-archive', key: 'X' }) + // A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave, but only under 100 MB without the article-size leg. + assert.include(rules.judge('k', [archive('Leaked Sex Video.rar', 'first.last88@gmail.com', 2, 100, 65_000_000)]), { rule: 'stub-archive', key: 'W' }) + assert.deepEqual(rules.judge('k', [archive('Leaked Sex Video.rar', 'first.last88@gmail.com', 2, 400, 260_000_000)]), { rule: null }) + // Not settled: a par2 can still join. + assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 2, 48, 12_000_000)], false), { rule: null }) + }) + + test('stub archives: small, single-group, split and large hash-named archives are kept', ({ assert }) => { + // An e-book: a one- or two-article release always has small articles (LOG.md (126)). + assert.deepEqual(rules.judge('k', [archive('Maeve Binchy - Tara Road (epub).rar', 'kie@home.now (KiE)', 2, 2, 521_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 1, 48, 12_000_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Some.Movie.2014.720p.BluRay-GRP.rar', 'poster@example.org', 2, 100, 70_000_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Some.Movie.2014.part01.rar', 'a@gmail.com', 2, 48, 12_000_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('cbcd32359460df1fa9b7d01d8720.rar', 'b648de0d1c0f7a9a77be273d@gmail.com', 2, 3163, 2_340_000_000)]), { rule: null }) + // A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)). + assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl.zip', 'home@home.com (TexasDj)', 2, 13, 3_300_000), groups: ['alt.binaries.sounds.karaoke', 'alt.binaries.test'] }]), { rule: null }) + // A rar with its par2 is no single archive. + assert.deepEqual(rules.judge('k', [archive('Chat Watch 1.2.1.zip', 'a@gmail.com', 2, 48, 12_000_000), file('Chat Watch 1.2.1.par2', 1, NYUU)]), { rule: null }) + const off = new RuleSet({ ...DEFAULT_RULES, dropStubArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] }) + assert.deepEqual(off.judge('k', [archive('Chat Watch 1.2.1.zip', 'jahnunsinos007@gmail.com', 2, 48, 12_000_000)]), { rule: null }) + }) + + test('nonsense pieces: one article of 100 kB to 1 MB under a release-like name', ({ assert }) => { + assert.include(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: 'nonsense-piece', key: 'nyuu' }) + // A 3 kB cue/log sidecar rar with a year in its name: under the floor. A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule. + assert.deepEqual(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar', 'fake@address.com (Psychopath)', 2, 1, 303_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 382_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar', 'saggir@domain.com', 1, 1, 113_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 2, 790_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null }) + const off = new RuleSet({ ...DEFAULT_RULES, dropNonsensePieces: false, samplePercent: 0, priorities: {}, droppedSignatures: [] }) + assert.deepEqual(off.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null }) + }) }) test.group('rules / upload tools', () => {