daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tiny-archive rule; the ladder's discard by index; the namer in chunks
Test and deploy / test (push) Successful in 3m1s
Details
Test and deploy / deploy (push) Successful in 32m21s
Details

Browse Source 
The tiny-archive rule (dropTinyArchives, both indexers): a settled single
archive of one article and 2-100 kB under a release-like name, spaces
allowed, outside the exempt groups and without an e-book, subtitle,
crack, cover or sidecar word: the 2015-17 exe-stub flood in miniature,
1.66M catalog rows (LOG.md (131), (133)).

PendingLadder.discard scanned every waiting row against the dropped
signatures and classes, 9+ minutes a sweep inside the claim, stalling
every lane: now driven from the dropped classes and signatures into the
indexes. The namer feeds the indexer the span's overview text chunk by
chunk (one string of it topped V8's limit) and gives up spans over 3M
articles. Migration 7000: a partial index on the taken probe candidates
for the prober's start.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-11-02 20:23:16 +01:00
1 parent 4d46db291e
commit aa4022295a
10 files changed
+267 -72
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

edge/src/lease.rs
+5
View File
@@ -77,6 +77,8 @@ pub struct RulesJson {
pub drop_stub_archives: Option<bool>,
#[serde(default)]
pub drop_nonsense_pieces: Option<bool>,
#[serde(default)]
pub drop_tiny_archives: Option<bool>,
}
impl RulesJson {
@@ -107,6 +109,9 @@ impl RulesJson {
if let Some(d) = self.drop_nonsense_pieces {
rules.drop_nonsense_pieces = d;
}
if let Some(d) = self.drop_tiny_archives {
rules.drop_tiny_archives = d;
}
rules
}
}
edge/src/names.rs
+66 -3
View File
@@ -1515,6 +1515,11 @@ static RELEASE_LIKE: LazyLock<Regex> = LazyLock::new(|| {
re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
});
/// RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way.
static RELEASE_LIKE_SPACED: LazyLock<Regex> = LazyLock::new(|| {
re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
});
/// The stub-archive rule (LOG.md (123)-(126), (128)): one unsplit archive, no par2, settled, posted to 2+ groups,
/// under STUB_MAX_BYTES (the fakes stop at ~90 MB; the hash-named uploads, real albums and passworded videos above
/// it don't), and its articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's
@@ -1528,6 +1533,8 @@ pub const STUB_WIDE_GROUPS: usize = 4;
/// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts).
pub const NONSENSE_MIN_BYTES: u64 = 100_000;
pub const NONSENSE_MAX_BYTES: u64 = 1_000_000;
/// The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars.
pub const TINY_MIN_BYTES: u64 = 2_000;
pub struct JudgedFile<'a> {
pub name: &'a str,
@@ -1589,6 +1596,27 @@ pub fn is_nonsense_piece(f: &JudgedFile) -> bool {
&& !is_random_file(f.name)
}
/// Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of
/// exe is the content), covers and sidecars.
static TINY_CONTENT: LazyLock<Regex> = LazyLock::new(|| {
re(r"(?i)\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b")
});
/// A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like,
/// unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book,
/// subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)).
pub fn is_tiny_archive(f: &JudgedFile) -> bool {
f.total_parts == 1
&& f.segments == 1
&& (TINY_MIN_BYTES..NONSENSE_MIN_BYTES).contains(&f.bytes)
&& !f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g))
&& test(&RELEASE_LIKE_SPACED, &file_stem(f.name))
&& !test(&EBOOK, f.name)
&& !test(&TINY_CONTENT, f.name)
&& !looks_scrambled(f.name)
&& !is_random_file(f.name)
}
pub enum Verdict {
Keep,
Drop { rule: &'static str, key: String, sample: bool },
@@ -1610,9 +1638,11 @@ pub struct Rules {
pub drop_executables: bool,
/// Fragments of the poster lowercased without punctuation (droppedPosters).
pub dropped_posters: Vec<String>,
/// The stub-archive rule (dropStubArchives) and the nonsense-piece rule (dropNonsensePieces).
/// The stub-archive rule (dropStubArchives), the nonsense-piece rule (dropNonsensePieces) and the tiny-archive
/// rule (dropTinyArchives).
pub drop_stub_archives: bool,
pub drop_nonsense_pieces: bool,
pub drop_tiny_archives: bool,
}
impl Rules {
@@ -1630,6 +1660,7 @@ impl Rules {
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
drop_stub_archives: true,
drop_nonsense_pieces: true,
drop_tiny_archives: true,
}
}
}
@@ -1665,6 +1696,9 @@ pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rul
if rules.drop_nonsense_pieces && is_nonsense_piece(f) {
return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample };
}
if rules.drop_tiny_archives && is_tiny_archive(f) {
return Verdict::Drop { rule: "tiny-archive", key: upload_tool(f.first_message_id).to_string(), sample };
}
}
if files.iter().any(|f| !is_random_file(f.name)) {
return Verdict::Keep;
@@ -1993,8 +2027,8 @@ mod single_archive_rule_tests {
let rules = Rules::defaults(Default::default());
// One 790 kB article under a release-like name: the newznzb.* pieces.
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
// A 3 kB cue/log sidecar rar with a year in its name, and a diet advert: under the floor.
assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), None);
// A 3 kB cue/log sidecar rar with a year in its name: under this rule's floor (the tiny-archive rule's outside the music groups).
assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
// A book (a year or a format in its name, written with spaces), a dotted one too: not a piece.
assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None);
assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None);
@@ -2006,6 +2040,35 @@ mod single_archive_rule_tests {
let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) };
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None);
}
#[test]
fn tiny_archives() {
let rules = Rules::defaults(Default::default());
// One article of 2-100 kB under a release name, dotted or with spaces: a RAR holding "<title>.exe" (LOG.md (131)).
assert_eq!(verdict("Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar", "Yanc@power-post.org <Yanc@power-post.org>", 1, 1, 15_084, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
assert_eq!(verdict("Cicero Magazin fuer politische Kultur Mai 05 2016.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 17_478, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
// A scene name without a year, a tag or a group suffix: not release-like, so kept (the flood's porn names mostly carry one).
assert_eq!(verdict("[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar", "Yanc@power-post.org", 2, 1, 23_449, true, &rules), None);
assert_eq!(verdict("Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 7_869, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
// Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives.
assert_eq!(verdict("The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar", "henk@example.org", 1, 1, 38_000, true, &rules), None);
assert_eq!(verdict("Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar", "poster@example.org", 1, 1, 60_000, true, &rules), None);
assert_eq!(verdict("Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None);
assert_eq!(verdict("Some Band - Album (2011) covers.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None);
assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 38_000, true, &rules), None);
// A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's.
let groups = vec!["alt.binaries.sounds.karaoke".to_string()];
let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl (2017).zip", total_parts: 1, groups: &groups, first_message_id: NYUU, poster: "home@home.com", segments: 1, bytes: 60_000 }];
assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep));
assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None);
assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None);
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 2, 20_000, true, &rules), None);
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 1_500, true, &rules), None);
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 150_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, false, &rules), None);
let off = Rules { drop_tiny_archives: false, ..Rules::defaults(Default::default()) };
assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, true, &off), None);
}
}
#[cfg(test)]