Single-archive rules: stub archives and nonsense pieces dropped at the edge
The judge gets each file's articles seen and bytes, and whether the window closed on the release. A settled release of one unsplit archive posted to 2+ groups is dropped as a stub archive when its articles are <= 256 KiB with the release >= 1 MB, or it is in 4+ groups, or by a webmail poster (those two under 100 MB); releases touching music, e-book, comics, tabs or fonts groups are exempt. A one-article archive of 100 kB-1 MB under a release-like, unscrambled name without spaces or an e-book format is dropped as a nonsense piece. Both switched by the rule set (dropStubArchives, dropNonsensePieces), mirrored in lib/rules, sampled like every drop (LOG.md (123)-(127)). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
2f1bbf2538
commit
e470e231d0
8 files changed
+397
-15
No files matched your search
+168
-4
@@ -1296,6 +1296,31 @@ pub fn is_sidecar(name: &str) -> bool {
|
||||
test(&SIDECAR, name)
|
||||
}
|
||||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||||
/// An unsplit archive (the judge's ARCHIVE and VOLUME): one zip, rar or 7z that is not a volume of a set.
|
||||
static ARCHIVE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(zip|rar|7z)$"));
|
||||
static VOLUME: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(part[0-9]+\.rar|r[0-9][0-9]|z[0-9][0-9]|7z\.[0-9][0-9][0-9]|[0-9][0-9][0-9])$"));
|
||||
/// A poster at a free mail provider (the judge's WEBMAIL): the stub floods' throwaway accounts.
|
||||
static WEBMAIL: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\.")
|
||||
});
|
||||
/// A name that reads as a release (the judge's RELEASE_LIKE): a year, a scene tag, a group suffix or dotted words.
|
||||
static RELEASE_LIKE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
|
||||
});
|
||||
|
||||
/// The stub-archive rule (LOG.md (123)-(126)): one unsplit archive, no par2, settled, posted to 2+ groups, and its
|
||||
/// articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's 250 kB default; a
|
||||
/// one-article release always has small articles, so e-books and sidecars are exempt), or 4+ groups, or a webmail
|
||||
/// poster, these two under STUB_MAX_BYTES (no article-size signal: the fakes stop at ~90 MB, the hash-named
|
||||
/// uploads and real albums above it don't).
|
||||
pub const STUB_ARTICLE_BYTES: u64 = 262_144;
|
||||
pub const STUB_MIN_BYTES: u64 = 1_000_000;
|
||||
pub const STUB_MAX_BYTES: u64 = 100_000_000;
|
||||
pub const STUB_WIDE_GROUPS: usize = 4;
|
||||
/// The nonsense-piece rule: one article holding one archive of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a
|
||||
/// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts).
|
||||
pub const NONSENSE_MIN_BYTES: u64 = 100_000;
|
||||
pub const NONSENSE_MAX_BYTES: u64 = 1_000_000;
|
||||
|
||||
pub struct JudgedFile<'a> {
|
||||
pub name: &'a str,
|
||||
@@ -1303,6 +1328,58 @@ pub struct JudgedFile<'a> {
|
||||
pub groups: &'a [String],
|
||||
pub first_message_id: &'a str,
|
||||
pub poster: &'a str,
|
||||
/// Articles seen and their bytes (the overview's :bytes).
|
||||
pub segments: u32,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
/// One unsplit archive and nothing else.
|
||||
fn single_archive<'a, 'b>(files: &'b [JudgedFile<'a>]) -> Option<&'b JudgedFile<'a>> {
|
||||
match files {
|
||||
[f] if test(&ARCHIVE, f.name) && !test(&VOLUME, f.name) => Some(f),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way
|
||||
/// the stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the
|
||||
/// stub-archive rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)).
|
||||
static STUB_EXEMPT_GROUPS: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)"));
|
||||
|
||||
/// The legs of the stub-archive rule a settled single archive trips ("A" article size, "X" 4+ groups, "W" webmail),
|
||||
/// empty when none.
|
||||
pub fn stub_archive_legs(f: &JudgedFile) -> String {
|
||||
let mut legs = String::new();
|
||||
if f.groups.len() < 2 || f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) {
|
||||
return legs;
|
||||
}
|
||||
let article = f.bytes / u64::from(f.segments.max(1));
|
||||
if article <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES {
|
||||
legs.push('A');
|
||||
}
|
||||
if f.groups.len() >= STUB_WIDE_GROUPS && f.bytes < STUB_MAX_BYTES {
|
||||
legs.push('X');
|
||||
}
|
||||
if f.bytes < STUB_MAX_BYTES && test(&WEBMAIL, f.poster) {
|
||||
legs.push('W');
|
||||
}
|
||||
legs
|
||||
}
|
||||
|
||||
/// An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece.
|
||||
static EBOOK: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b"));
|
||||
|
||||
/// A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the
|
||||
/// newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces).
|
||||
pub fn is_nonsense_piece(f: &JudgedFile) -> bool {
|
||||
f.total_parts == 1
|
||||
&& f.segments == 1
|
||||
&& (NONSENSE_MIN_BYTES..NONSENSE_MAX_BYTES).contains(&f.bytes)
|
||||
&& !f.name.contains(' ')
|
||||
&& test(&RELEASE_LIKE, &file_stem(f.name))
|
||||
&& !test(&EBOOK, f.name)
|
||||
&& !looks_scrambled(f.name)
|
||||
&& !is_random_file(f.name)
|
||||
}
|
||||
|
||||
pub enum Verdict {
|
||||
@@ -1324,6 +1401,9 @@ pub struct Rules {
|
||||
pub drop_executables: bool,
|
||||
/// Fragments of the poster lowercased without punctuation (droppedPosters).
|
||||
pub dropped_posters: Vec<String>,
|
||||
/// The stub-archive rule (dropStubArchives) and the nonsense-piece rule (dropNonsensePieces).
|
||||
pub drop_stub_archives: bool,
|
||||
pub drop_nonsense_pieces: bool,
|
||||
}
|
||||
|
||||
impl Rules {
|
||||
@@ -1338,6 +1418,8 @@ impl Rules {
|
||||
sample_percent: 1.0,
|
||||
drop_executables: true,
|
||||
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
|
||||
drop_stub_archives: true,
|
||||
drop_nonsense_pieces: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1347,8 +1429,9 @@ pub fn poster_words(poster: &str) -> String {
|
||||
poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect()
|
||||
}
|
||||
|
||||
/// RuleSet.judge.
|
||||
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
||||
/// RuleSet.judge. `settled`: the window closed on the release (a par2 can still join an open one), which the
|
||||
/// single-archive rules need.
|
||||
pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rules) -> Verdict {
|
||||
let sample = explored(release_key, rules.sample_percent);
|
||||
if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) {
|
||||
let words = poster_words(first.poster);
|
||||
@@ -1362,6 +1445,17 @@ pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict
|
||||
let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
|
||||
return Verdict::Drop { rule: "executable", key, sample };
|
||||
}
|
||||
if let (true, Some(f)) = (settled, single_archive(files)) {
|
||||
if rules.drop_stub_archives {
|
||||
let legs = stub_archive_legs(f);
|
||||
if !legs.is_empty() {
|
||||
return Verdict::Drop { rule: "stub-archive", key: legs, sample };
|
||||
}
|
||||
}
|
||||
if rules.drop_nonsense_pieces && is_nonsense_piece(f) {
|
||||
return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample };
|
||||
}
|
||||
}
|
||||
if files.iter().any(|f| !is_random_file(f.name)) {
|
||||
return Verdict::Keep;
|
||||
}
|
||||
@@ -1567,8 +1661,8 @@ mod poster_rule_tests {
|
||||
|
||||
fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> {
|
||||
let groups = vec!["alt.binaries.erotica".to_string()];
|
||||
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster }];
|
||||
match judge("k", &files, rules) {
|
||||
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster, segments: 1, bytes: 200_000 }];
|
||||
match judge("k", &files, true, rules) {
|
||||
Verdict::Keep => None,
|
||||
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||
}
|
||||
@@ -1588,6 +1682,76 @@ mod poster_rule_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod single_archive_rule_tests {
|
||||
use super::{judge, JudgedFile, Rules, Verdict};
|
||||
|
||||
const NYUU: &str = "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu";
|
||||
|
||||
fn verdict(name: &str, poster: &str, groups: usize, segments: u32, bytes: u64, settled: bool, rules: &Rules) -> Option<(&'static str, String)> {
|
||||
let groups: Vec<String> = (0..groups).map(|i| format!("alt.binaries.g{i}")).collect();
|
||||
let files = [JudgedFile { name, total_parts: segments, groups: &groups, first_message_id: NYUU, poster, segments, bytes }];
|
||||
match judge("k", &files, settled, rules) {
|
||||
Verdict::Keep => None,
|
||||
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_archives_by_article_size_groups_or_webmail() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave.
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com (Jahnunsinos007)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "AW".to_string())));
|
||||
assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "Yenc@power-post.org (Yenc-PP-A&A)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "A".to_string())));
|
||||
// Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone).
|
||||
assert_eq!(verdict("Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar", "Yenc@power-post.org (Yenc-PP-A&A)", 5, 2, 319_000, true, &rules), Some(("stub-archive", "X".to_string())));
|
||||
// A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave.
|
||||
assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 400, 260_000_000, true, &rules), None, "over 100 MB without the article-size leg");
|
||||
assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 100, 65_000_000, true, &rules), Some(("stub-archive", "W".to_string())));
|
||||
// Settled only: a par2 can still join an open release.
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, false, &rules), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_and_single_group_and_split_archives_are_kept() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// An e-book: one or two small articles always have small articles (LOG.md (126)).
|
||||
assert_eq!(verdict("Maeve Binchy - Tara Road (epub).rar", "kie@home.now (KiE)", 2, 2, 521_000, true, &rules), None);
|
||||
// One group: most real single rars.
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 1, 48, 12_000_000, true, &rules), None);
|
||||
// Modern 700 kB articles, no webmail, two groups.
|
||||
assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "poster@example.org", 2, 100, 70_000_000, true, &rules), None);
|
||||
// A volume, not a single archive; a hash-named 2 GB upload from a gmail-looking poster.
|
||||
assert_eq!(verdict("Some.Movie.2014.part01.rar", "a@gmail.com", 2, 48, 12_000_000, true, &rules), None);
|
||||
assert_eq!(verdict("cbcd32359460df1fa9b7d01d8720.rar", "b648de0d1c0f7a9a77be273d@gmail.com", 2, 3163, 2_340_000_000, true, &rules), None);
|
||||
let off = Rules { drop_stub_archives: false, ..Rules::defaults(Default::default()) };
|
||||
assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, true, &off), None);
|
||||
// A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)).
|
||||
let groups = vec!["alt.binaries.sounds.karaoke".to_string(), "alt.binaries.test".to_string()];
|
||||
let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl.zip", total_parts: 13, groups: &groups, first_message_id: NYUU, poster: "home@home.com (TexasDj)", segments: 13, bytes: 3_300_000 }];
|
||||
assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonsense_pieces() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
// One 790 kB article under a release-like name: the newznzb.* pieces.
|
||||
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
|
||||
// A 3 kB cue/log sidecar rar with a year in its name, and a diet advert: under the floor.
|
||||
assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), None);
|
||||
// A book (a year or a format in its name, written with spaces), a dotted one too: not a piece.
|
||||
assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None);
|
||||
assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None);
|
||||
assert_eq!(verdict("Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar", "saggir@domain.com", 1, 1, 113_000, true, &rules), None);
|
||||
// Two articles, or no release-like name, or scrambled: not this rule.
|
||||
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 2, 790_000, true, &rules), None);
|
||||
assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None);
|
||||
assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None);
|
||||
let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) };
|
||||
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod image_set_tests {
|
||||
use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};
|
||||
|
||||
Reference in new issue
Block a user