Single-archive rules: stub archives and nonsense pieces dropped at the edge

The judge gets each file's articles seen and bytes, and whether the window
closed on the release. A settled release of one unsplit archive posted to 2+
groups is dropped as a stub archive when its articles are <= 256 KiB with the
release >= 1 MB, or it is in 4+ groups, or by a webmail poster (those two
under 100 MB); releases touching music, e-book, comics, tabs or fonts groups
are exempt. A one-article archive of 100 kB-1 MB under a release-like,
unscrambled name without spaces or an e-book format is dropped as a nonsense
piece. Both switched by the rule set (dropStubArchives, dropNonsensePieces),
mirrored in lib/rules, sampled like every drop (LOG.md (123)-(127)).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

edge/src/indexer.rs

@@ -1423,9 +1423,21 @@ impl RangeIndexer {
     fn write(&mut self, key: &str, upload: usize, list: Vec<Vec<Seg>>, finished: bool, edge: bool, album: Option<String>) {
         let judged: Vec<JudgedFile> = list
             .iter()
-            .map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster })
+            .map(|f| JudgedFile {
+                name: &f[0].file.name,
+                total_parts: f[0].file.total_parts,
+                groups: &f[0].file.groups,
+                first_message_id: &f[0].message_id,
+                poster: &f[0].file.poster,
+                segments: f.len() as u32,
+                bytes: f.iter().map(|s| s.bytes.max(0) as u64).sum(),
+            })
             .collect();
-        match names::judge(key, &judged, &self.rules) {
+        // Settled for the single-archive rules: the window closed on it (a par2 can still join an open one). The
+        // start edge (posted within the window of the block's start) doesn't count: a par2 comes after its rar, so
+        // one in the previous block is rare (0 par2 siblings of 3,000 catalog hits, LOG.md (125)), and 7% of the
+        // catalog's releases are start edges, which would escape the rule.
+        match names::judge(key, &judged, finished, &self.rules) {
             Verdict::Keep => {
                 let candidates = candidates_of(key, &list, upload);
                 let mut chunk = chunk_of(key, &list, upload);

edge/src/lease.rs

@@ -71,6 +71,10 @@ pub struct RulesJson {
     pub drop_executables: Option<bool>,
     #[serde(default)]
     pub dropped_posters: Option<Vec<String>>,
+    #[serde(default)]
+    pub drop_stub_archives: Option<bool>,
+    #[serde(default)]
+    pub drop_nonsense_pieces: Option<bool>,
 }
 
 impl RulesJson {
@@ -94,6 +98,12 @@ impl RulesJson {
         if let Some(posters) = self.dropped_posters {
             rules.dropped_posters = posters;
         }
+        if let Some(d) = self.drop_stub_archives {
+            rules.drop_stub_archives = d;
+        }
+        if let Some(d) = self.drop_nonsense_pieces {
+            rules.drop_nonsense_pieces = d;
+        }
         rules
     }
 }

edge/src/names.rs

@@ -1296,6 +1296,31 @@ pub fn is_sidecar(name: &str) -> bool {
     test(&SIDECAR, name)
 }
 static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
+/// An unsplit archive (the judge's ARCHIVE and VOLUME): one zip, rar or 7z that is not a volume of a set.
+static ARCHIVE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(zip|rar|7z)$"));
+static VOLUME: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(part[0-9]+\.rar|r[0-9][0-9]|z[0-9][0-9]|7z\.[0-9][0-9][0-9]|[0-9][0-9][0-9])$"));
+/// A poster at a free mail provider (the judge's WEBMAIL): the stub floods' throwaway accounts.
+static WEBMAIL: LazyLock<Regex> = LazyLock::new(|| {
+    re(r"(?i)@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\.")
+});
+/// A name that reads as a release (the judge's RELEASE_LIKE): a year, a scene tag, a group suffix or dotted words.
+static RELEASE_LIKE: LazyLock<Regex> = LazyLock::new(|| {
+    re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
+});
+
+/// The stub-archive rule (LOG.md (123)-(126)): one unsplit archive, no par2, settled, posted to 2+ groups, and its
+/// articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's 250 kB default; a
+/// one-article release always has small articles, so e-books and sidecars are exempt), or 4+ groups, or a webmail
+/// poster, these two under STUB_MAX_BYTES (no article-size signal: the fakes stop at ~90 MB, the hash-named
+/// uploads and real albums above it don't).
+pub const STUB_ARTICLE_BYTES: u64 = 262_144;
+pub const STUB_MIN_BYTES: u64 = 1_000_000;
+pub const STUB_MAX_BYTES: u64 = 100_000_000;
+pub const STUB_WIDE_GROUPS: usize = 4;
+/// The nonsense-piece rule: one article holding one archive of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a
+/// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts).
+pub const NONSENSE_MIN_BYTES: u64 = 100_000;
+pub const NONSENSE_MAX_BYTES: u64 = 1_000_000;
 
 pub struct JudgedFile<'a> {
     pub name: &'a str,
@@ -1303,6 +1328,58 @@ pub struct JudgedFile<'a> {
     pub groups: &'a [String],
     pub first_message_id: &'a str,
     pub poster: &'a str,
+    /// Articles seen and their bytes (the overview's :bytes).
+    pub segments: u32,
+    pub bytes: u64,
+}
+
+/// One unsplit archive and nothing else.
+fn single_archive<'a, 'b>(files: &'b [JudgedFile<'a>]) -> Option<&'b JudgedFile<'a>> {
+    match files {
+        [f] if test(&ARCHIVE, f.name) && !test(&VOLUME, f.name) => Some(f),
+        _ => None,
+    }
+}
+
+/// Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way
+/// the stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the
+/// stub-archive rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)).
+static STUB_EXEMPT_GROUPS: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)"));
+
+/// The legs of the stub-archive rule a settled single archive trips ("A" article size, "X" 4+ groups, "W" webmail),
+/// empty when none.
+pub fn stub_archive_legs(f: &JudgedFile) -> String {
+    let mut legs = String::new();
+    if f.groups.len() < 2 || f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) {
+        return legs;
+    }
+    let article = f.bytes / u64::from(f.segments.max(1));
+    if article <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES {
+        legs.push('A');
+    }
+    if f.groups.len() >= STUB_WIDE_GROUPS && f.bytes < STUB_MAX_BYTES {
+        legs.push('X');
+    }
+    if f.bytes < STUB_MAX_BYTES && test(&WEBMAIL, f.poster) {
+        legs.push('W');
+    }
+    legs
+}
+
+/// An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece.
+static EBOOK: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b"));
+
+/// A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the
+/// newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces).
+pub fn is_nonsense_piece(f: &JudgedFile) -> bool {
+    f.total_parts == 1
+        && f.segments == 1
+        && (NONSENSE_MIN_BYTES..NONSENSE_MAX_BYTES).contains(&f.bytes)
+        && !f.name.contains(' ')
+        && test(&RELEASE_LIKE, &file_stem(f.name))
+        && !test(&EBOOK, f.name)
+        && !looks_scrambled(f.name)
+        && !is_random_file(f.name)
 }
 
 pub enum Verdict {
@@ -1324,6 +1401,9 @@ pub struct Rules {
     pub drop_executables: bool,
     /// Fragments of the poster lowercased without punctuation (droppedPosters).
     pub dropped_posters: Vec<String>,
+    /// The stub-archive rule (dropStubArchives) and the nonsense-piece rule (dropNonsensePieces).
+    pub drop_stub_archives: bool,
+    pub drop_nonsense_pieces: bool,
 }
 
 impl Rules {
@@ -1338,6 +1418,8 @@ impl Rules {
             sample_percent: 1.0,
             drop_executables: true,
             dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
+            drop_stub_archives: true,
+            drop_nonsense_pieces: true,
         }
     }
 }
@@ -1347,8 +1429,9 @@ pub fn poster_words(poster: &str) -> String {
     poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect()
 }
 
-/// RuleSet.judge.
-pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
+/// RuleSet.judge. `settled`: the window closed on the release (a par2 can still join an open one), which the
+/// single-archive rules need.
+pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rules) -> Verdict {
     let sample = explored(release_key, rules.sample_percent);
     if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) {
         let words = poster_words(first.poster);
@@ -1362,6 +1445,17 @@ pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict
         let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
         return Verdict::Drop { rule: "executable", key, sample };
     }
+    if let (true, Some(f)) = (settled, single_archive(files)) {
+        if rules.drop_stub_archives {
+            let legs = stub_archive_legs(f);
+            if !legs.is_empty() {
+                return Verdict::Drop { rule: "stub-archive", key: legs, sample };
+            }
+        }
+        if rules.drop_nonsense_pieces && is_nonsense_piece(f) {
+            return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample };
+        }
+    }
     if files.iter().any(|f| !is_random_file(f.name)) {
         return Verdict::Keep;
     }
@@ -1567,8 +1661,8 @@ mod poster_rule_tests {
 
     fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> {
         let groups = vec!["alt.binaries.erotica".to_string()];
-        let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster }];
-        match judge("k", &files, rules) {
+        let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster, segments: 1, bytes: 200_000 }];
+        match judge("k", &files, true, rules) {
             Verdict::Keep => None,
             Verdict::Drop { rule, key, .. } => Some((rule, key)),
         }
@@ -1588,6 +1682,76 @@ mod poster_rule_tests {
     }
 }
 
+#[cfg(test)]
+mod single_archive_rule_tests {
+    use super::{judge, JudgedFile, Rules, Verdict};
+
+    const NYUU: &str = "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu";
+
+    fn verdict(name: &str, poster: &str, groups: usize, segments: u32, bytes: u64, settled: bool, rules: &Rules) -> Option<(&'static str, String)> {
+        let groups: Vec<String> = (0..groups).map(|i| format!("alt.binaries.g{i}")).collect();
+        let files = [JudgedFile { name, total_parts: segments, groups: &groups, first_message_id: NYUU, poster, segments, bytes }];
+        match judge("k", &files, settled, rules) {
+            Verdict::Keep => None,
+            Verdict::Drop { rule, key, .. } => Some((rule, key)),
+        }
+    }
+
+    #[test]
+    fn stub_archives_by_article_size_groups_or_webmail() {
+        let rules = Rules::defaults(Default::default());
+        // Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave.
+        assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com (Jahnunsinos007)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "AW".to_string())));
+        assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "Yenc@power-post.org (Yenc-PP-A&A)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "A".to_string())));
+        // Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone).
+        assert_eq!(verdict("Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar", "Yenc@power-post.org (Yenc-PP-A&A)", 5, 2, 319_000, true, &rules), Some(("stub-archive", "X".to_string())));
+        // A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave.
+        assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 400, 260_000_000, true, &rules), None, "over 100 MB without the article-size leg");
+        assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 100, 65_000_000, true, &rules), Some(("stub-archive", "W".to_string())));
+        // Settled only: a par2 can still join an open release.
+        assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, false, &rules), None);
+    }
+
+    #[test]
+    fn small_and_single_group_and_split_archives_are_kept() {
+        let rules = Rules::defaults(Default::default());
+        // An e-book: one or two small articles always have small articles (LOG.md (126)).
+        assert_eq!(verdict("Maeve Binchy - Tara Road (epub).rar", "kie@home.now (KiE)", 2, 2, 521_000, true, &rules), None);
+        // One group: most real single rars.
+        assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 1, 48, 12_000_000, true, &rules), None);
+        // Modern 700 kB articles, no webmail, two groups.
+        assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "poster@example.org", 2, 100, 70_000_000, true, &rules), None);
+        // A volume, not a single archive; a hash-named 2 GB upload from a gmail-looking poster.
+        assert_eq!(verdict("Some.Movie.2014.part01.rar", "a@gmail.com", 2, 48, 12_000_000, true, &rules), None);
+        assert_eq!(verdict("cbcd32359460df1fa9b7d01d8720.rar", "b648de0d1c0f7a9a77be273d@gmail.com", 2, 3163, 2_340_000_000, true, &rules), None);
+        let off = Rules { drop_stub_archives: false, ..Rules::defaults(Default::default()) };
+        assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, true, &off), None);
+        // A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)).
+        let groups = vec!["alt.binaries.sounds.karaoke".to_string(), "alt.binaries.test".to_string()];
+        let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl.zip", total_parts: 13, groups: &groups, first_message_id: NYUU, poster: "home@home.com (TexasDj)", segments: 13, bytes: 3_300_000 }];
+        assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep));
+    }
+
+    #[test]
+    fn nonsense_pieces() {
+        let rules = Rules::defaults(Default::default());
+        // One 790 kB article under a release-like name: the newznzb.* pieces.
+        assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
+        // A 3 kB cue/log sidecar rar with a year in its name, and a diet advert: under the floor.
+        assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), None);
+        // A book (a year or a format in its name, written with spaces), a dotted one too: not a piece.
+        assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None);
+        assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None);
+        assert_eq!(verdict("Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar", "saggir@domain.com", 1, 1, 113_000, true, &rules), None);
+        // Two articles, or no release-like name, or scrambled: not this rule.
+        assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 2, 790_000, true, &rules), None);
+        assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None);
+        assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None);
+        let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) };
+        assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None);
+    }
+}
+
 #[cfg(test)]
 mod image_set_tests {
     use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};