diff --git a/REBUILD.md b/REBUILD.md index 7eab73d..02e2fff 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -806,9 +806,17 @@ unscrambled name without spaces and without an e-book format in it is dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool): the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of the catalog; under 100 kB are cue/log sidecars and adverts, and the names -with spaces are books and albums. Both rules are switched by the -rule set (`dropStubArchives`, `dropNonsensePieces`), sampled like every -drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer +with spaces are books and albums. Below that, a settled single archive of +one article and 2–100 kB under a release-like name (spaces allowed), outside +the exempt groups and without an e-book, subtitle, crack/keygen/trainer, +cover or sidecar word, is dropped as a **tiny archive** (`tiny-archive`, +key: the upload tool; LOG.md (131), (133)): the 2015–17 flood's RARs +holding ".exe" at 2–60 kB, 1.66M catalog rows from a dozen posters +(12 of 12 fetched fake); the word exemptions keep the real tiny archives +(subtitles, cracks, trainers) at the price of ~50k fakes named "Incl +Keygen" that the prober judges later. The rules are switched by the +rule set (`dropStubArchives`, `dropNonsensePieces`, `dropTinyArchives`), +sampled like every drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer judges as each batch completes files, so it passes settled without the window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now), `check.ts` and the fixture comparison (the parity with the TypeScript diff --git a/app/services/pending_ladder.ts b/app/services/pending_ladder.ts index 5328dfe..2ef22d2 100644 --- a/app/services/pending_ladder.ts +++ b/app/services/pending_ladder.ts @@ -375,21 +375,28 @@ class PendingLadder { ) } - /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */ + /** + * Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how + * many. Driven from the dropped classes and signatures into the claim's index and the ladder's: a scan of the + * waiting rows testing each (44M on 2026-11-02) ran 9+ minutes on the HDD every sweep (LOG.md (133)). + */ async discard(limit = 10_000): Promise<number> { const { samplePercent } = await RuleSets.current() - const { rows } = await db.rawQuery( - `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p - WHERE p.status = ? AND (EXISTS ( - SELECT 1 FROM pending_signatures s - WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))) - OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass)) - LIMIT ?`, + const { rows: byClass } = await db.rawQuery( + `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_classes c + JOIN pending_releases p ON p.status = ? AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass + WHERE c.dropped LIMIT ?`, [PENDING.waiting, limit] ) - const keys = rows - .map((r: any) => r.release_key as string) - .filter((key: string) => !explored(key, samplePercent)) + const { rows: bySignature } = await db.rawQuery( + `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_signatures s + JOIN pending_releases p ON p.status = ? AND p.group_id = s.group_id AND p.tool = s.tool AND (s.poster = '' OR s.poster = coalesce(p.poster, '')) + WHERE s.dropped LIMIT ?`, + [PENDING.waiting, limit] + ) + const keys = [ + ...new Set([...byClass, ...bySignature].map((r: any) => r.release_key as string)), + ].filter((key: string) => !explored(key, samplePercent)) if (keys.length === 0) return 0 const { rows: gone } = await db.rawQuery( `UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k) diff --git a/app/services/pending_namer.ts b/app/services/pending_namer.ts index 6c8c126..e5a994c 100644 --- a/app/services/pending_namer.ts +++ b/app/services/pending_namer.ts @@ -30,6 +30,13 @@ interface NamedRow { /** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */ const MAX_MOVES = 3 +/** + * The widest article span the namer reads again (a busy group posts 50M articles a day, a large upload among + * them spans a million or two): one wider is a name the edge saw again much later ("v-lgpc.3ds"), and its span + * of segments would not fit the indexer's memory (LOG.md (133)). + */ +const MAX_SPAN = 3_000_000 + /** * The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments * read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one @@ -94,6 +101,10 @@ export class PendingNamer { /** One named release into the catalog. */ async move(row: NamedRow): Promise<void> { + if (row.last_article - row.first_article > MAX_SPAN) + throw new Error( + `its articles ${row.first_article}–${row.last_article} span more than ${MAX_SPAN}` + ) const group = await db.from('groups').where('id', row.group_id).firstOrFail() const range = await db .from('ranges') @@ -103,19 +114,20 @@ export class PendingNamer { .orderBy('id') .first() if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`) - const text = await this.overviewText(group.name, row) const rules = await RuleSets.current() const ix = new RangeIndexer({ group: group.name, start: row.first_article, end: row.last_article, rules: { ...rules, samplePercent: 0 }, - idleChunks: 0, + // No file given up before the end: the span is read chunk by chunk (one text of it topped V8's string). + idleChunks: Number.MAX_SAFE_INTEGER, edgeArticles: 0, // The edge wrote it partial when its file is short of parts on the server: so must the re-read. partialFiles: true, }) - ix.addText(text, row.last_article + 1) + for await (const piece of this.overviewPieces(group.name, row)) + ix.addText(piece.text, piece.next) ix.finish() const batch = ix.take() const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key)) @@ -160,50 +172,20 @@ export class PendingNamer { }) } - /** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */ - private async overviewText(groupName: string, row: NamedRow): Promise<string> { - if (this.corpusDir) { - const stored = await db - .from('ranges') - .where('group_id', row.group_id) - .whereNotNull('stored_at') - .where('start_id', '<=', row.last_article) - .where('end_id', '>=', row.first_article) - .orderBy('start_id') - const parts: string[] = [] - let covered = true - for (const r of stored) { - const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) } - const size = coordinatorConfig.chunkArticles - for ( - let from = - range.start + - Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size; - from <= Math.min(row.last_article, range.end); - from += size - ) { - const raw = await Corpus.read( - range, - { from, to: Math.min(from + size - 1, range.end) }, - this.corpusDir - ) - if (!raw) { - covered = false - break - } - parts.push((await inflate(raw)).toString('latin1')) - } - if (!covered) break - } - if ( - covered && - parts.length > 0 && - stored.length > 0 && - Number(stored[0].start_id) <= row.first_article && - Number(stored[stored.length - 1].end_id) >= row.last_article - ) { - return parts.join('') - } + /** + * The overview lines of the release's article span, a chunk of articles at a time with the first article after + * it: the stored corpus chunks when its ranges are stored and contiguous, else XOVER in windows of a chunk. + */ + private async *overviewPieces( + groupName: string, + row: NamedRow + ): AsyncGenerator<{ text: string; next: number }> { + const size = coordinatorConfig.chunkArticles + const stored = await this.storedChunks(groupName, row) + if (stored) { + for (const { raw, next } of stored) + yield { text: (await inflate(raw)).toString('latin1'), next } + return } this.client ??= await XoverClient.connect({ host: env.get('NNTP_HOST')!, @@ -217,13 +199,55 @@ export class PendingNamer { this.usedAt = Date.now() try { await this.client.group(groupName) - // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so). - const raw = await this.client.xoverRaw(row.first_article, row.last_article) - return (await inflate(raw)).toString('latin1') + for (let from = row.first_article; from <= row.last_article; from += size) { + const to = Math.min(from + size - 1, row.last_article) + // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so). + const raw = await this.client.xoverRaw(from, to) + yield { text: (await inflate(raw)).toString('latin1'), next: to + 1 } + } } catch (error) { this.client.close() this.client = null throw error } } + + /** The span's stored corpus chunks (compressed, with the first article after each), or null when any is missing. */ + private async storedChunks( + groupName: string, + row: NamedRow + ): Promise<{ raw: Buffer; next: number }[] | null> { + if (!this.corpusDir) return null + const stored = await db + .from('ranges') + .where('group_id', row.group_id) + .whereNotNull('stored_at') + .where('start_id', '<=', row.last_article) + .where('end_id', '>=', row.first_article) + .orderBy('start_id') + if ( + stored.length === 0 || + Number(stored[0].start_id) > row.first_article || + Number(stored[stored.length - 1].end_id) < row.last_article + ) + return null + const size = coordinatorConfig.chunkArticles + const chunks: { raw: Buffer; next: number }[] = [] + for (const r of stored) { + const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) } + for ( + let from = + range.start + + Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size; + from <= Math.min(row.last_article, range.end); + from += size + ) { + const to = Math.min(from + size - 1, range.end) + const raw = await Corpus.read(range, { from, to }, this.corpusDir) + if (!raw) return null + chunks.push({ raw, next: to + 1 }) + } + } + return chunks + } } diff --git a/database/migrations/1770000007000_probe_candidates_taken_index.ts b/database/migrations/1770000007000_probe_candidates_taken_index.ts new file mode 100644 index 0000000..5d34469 --- /dev/null +++ b/database/migrations/1770000007000_probe_candidates_taken_index.ts @@ -0,0 +1,20 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The prober's start puts a previous run's claims back (probe_candidates status taken → waiting), which scanned + * the 7.8 GB candidates table on the HDD for 4 minutes after every deploy (LOG.md (133)): a partial index on the + * few taken rows makes it instant. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + this.schema.raw( + `CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_taken_idx ON probe_candidates (id) WHERE status = 'taken'` + ) + } + + async down() { + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_taken_idx`) + } +} diff --git a/edge/src/lease.rs b/edge/src/lease.rs index 44e0adb..07278fa 100644 --- a/edge/src/lease.rs +++ b/edge/src/lease.rs @@ -77,6 +77,8 @@ pub struct RulesJson { pub drop_stub_archives: Option<bool>, #[serde(default)] pub drop_nonsense_pieces: Option<bool>, + #[serde(default)] + pub drop_tiny_archives: Option<bool>, } impl RulesJson { @@ -107,6 +109,9 @@ impl RulesJson { if let Some(d) = self.drop_nonsense_pieces { rules.drop_nonsense_pieces = d; } + if let Some(d) = self.drop_tiny_archives { + rules.drop_tiny_archives = d; + } rules } } diff --git a/edge/src/names.rs b/edge/src/names.rs index 70a26bd..5255448 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -1515,6 +1515,11 @@ static RELEASE_LIKE: LazyLock<Regex> = LazyLock::new(|| { re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)") }); +/// RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way. +static RELEASE_LIKE_SPACED: LazyLock<Regex> = LazyLock::new(|| { + re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)") +}); + /// The stub-archive rule (LOG.md (123)-(126), (128)): one unsplit archive, no par2, settled, posted to 2+ groups, /// under STUB_MAX_BYTES (the fakes stop at ~90 MB; the hash-named uploads, real albums and passworded videos above /// it don't), and its articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's @@ -1528,6 +1533,8 @@ pub const STUB_WIDE_GROUPS: usize = 4; /// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts). pub const NONSENSE_MIN_BYTES: u64 = 100_000; pub const NONSENSE_MAX_BYTES: u64 = 1_000_000; +/// The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars. +pub const TINY_MIN_BYTES: u64 = 2_000; pub struct JudgedFile<'a> { pub name: &'a str, @@ -1589,6 +1596,27 @@ pub fn is_nonsense_piece(f: &JudgedFile) -> bool { && !is_random_file(f.name) } +/// Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of +/// exe is the content), covers and sidecars. +static TINY_CONTENT: LazyLock<Regex> = LazyLock::new(|| { + re(r"(?i)\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b") +}); + +/// A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like, +/// unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book, +/// subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)). +pub fn is_tiny_archive(f: &JudgedFile) -> bool { + f.total_parts == 1 + && f.segments == 1 + && (TINY_MIN_BYTES..NONSENSE_MIN_BYTES).contains(&f.bytes) + && !f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) + && test(&RELEASE_LIKE_SPACED, &file_stem(f.name)) + && !test(&EBOOK, f.name) + && !test(&TINY_CONTENT, f.name) + && !looks_scrambled(f.name) + && !is_random_file(f.name) +} + pub enum Verdict { Keep, Drop { rule: &'static str, key: String, sample: bool }, @@ -1610,9 +1638,11 @@ pub struct Rules { pub drop_executables: bool, /// Fragments of the poster lowercased without punctuation (droppedPosters). pub dropped_posters: Vec<String>, - /// The stub-archive rule (dropStubArchives) and the nonsense-piece rule (dropNonsensePieces). + /// The stub-archive rule (dropStubArchives), the nonsense-piece rule (dropNonsensePieces) and the tiny-archive + /// rule (dropTinyArchives). pub drop_stub_archives: bool, pub drop_nonsense_pieces: bool, + pub drop_tiny_archives: bool, } impl Rules { @@ -1630,6 +1660,7 @@ impl Rules { dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(), drop_stub_archives: true, drop_nonsense_pieces: true, + drop_tiny_archives: true, } } } @@ -1665,6 +1696,9 @@ pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rul if rules.drop_nonsense_pieces && is_nonsense_piece(f) { return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample }; } + if rules.drop_tiny_archives && is_tiny_archive(f) { + return Verdict::Drop { rule: "tiny-archive", key: upload_tool(f.first_message_id).to_string(), sample }; + } } if files.iter().any(|f| !is_random_file(f.name)) { return Verdict::Keep; @@ -1993,8 +2027,8 @@ mod single_archive_rule_tests { let rules = Rules::defaults(Default::default()); // One 790 kB article under a release-like name: the newznzb.* pieces. assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string()))); - // A 3 kB cue/log sidecar rar with a year in its name, and a diet advert: under the floor. - assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), None); + // A 3 kB cue/log sidecar rar with a year in its name: under this rule's floor (the tiny-archive rule's outside the music groups). + assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); // A book (a year or a format in its name, written with spaces), a dotted one too: not a piece. assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None); assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None); @@ -2006,6 +2040,35 @@ mod single_archive_rule_tests { let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) }; assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None); } + + #[test] + fn tiny_archives() { + let rules = Rules::defaults(Default::default()); + // One article of 2-100 kB under a release name, dotted or with spaces: a RAR holding "<title>.exe" (LOG.md (131)). + assert_eq!(verdict("Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar", "Yanc@power-post.org <Yanc@power-post.org>", 1, 1, 15_084, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); + assert_eq!(verdict("Cicero Magazin fuer politische Kultur Mai 05 2016.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 17_478, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); + // A scene name without a year, a tag or a group suffix: not release-like, so kept (the flood's porn names mostly carry one). + assert_eq!(verdict("[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar", "Yanc@power-post.org", 2, 1, 23_449, true, &rules), None); + assert_eq!(verdict("Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 7_869, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); + // Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives. + assert_eq!(verdict("The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar", "henk@example.org", 1, 1, 38_000, true, &rules), None); + assert_eq!(verdict("Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar", "poster@example.org", 1, 1, 60_000, true, &rules), None); + assert_eq!(verdict("Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None); + assert_eq!(verdict("Some Band - Album (2011) covers.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None); + assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 38_000, true, &rules), None); + // A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's. + let groups = vec!["alt.binaries.sounds.karaoke".to_string()]; + let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl (2017).zip", total_parts: 1, groups: &groups, first_message_id: NYUU, poster: "home@home.com", segments: 1, bytes: 60_000 }]; + assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep)); + assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None); + assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None); + assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 2, 20_000, true, &rules), None); + assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 1_500, true, &rules), None); + assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 150_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string()))); + assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, false, &rules), None); + let off = Rules { drop_tiny_archives: false, ..Rules::defaults(Default::default()) }; + assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, true, &off), None); + } } #[cfg(test)] diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 40e6620..b21ffa2 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -66,7 +66,7 @@ export interface ProbeCandidate { /** What a rule dropped in a batch (REBUILD.md, rules and recovery). */ export interface RuleCount { - rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'incomplete' + rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive' | 'incomplete' key: string articles: number files: number diff --git a/lib/rules/index.ts b/lib/rules/index.ts index a292266..5b810e3 100644 --- a/lib/rules/index.ts +++ b/lib/rules/index.ts @@ -1,2 +1,2 @@ export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js' -export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, stubArchiveLegs, isNonsensePiece, STUB_ARTICLE_BYTES, STUB_MIN_BYTES, STUB_MAX_BYTES, STUB_WIDE_GROUPS, NONSENSE_MIN_BYTES, NONSENSE_MAX_BYTES, type Rules, type Verdict, type JudgedFile } from './rule_set.js' +export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, stubArchiveLegs, isNonsensePiece, isTinyArchive, STUB_ARTICLE_BYTES, STUB_MIN_BYTES, STUB_MAX_BYTES, STUB_WIDE_GROUPS, NONSENSE_MIN_BYTES, NONSENSE_MAX_BYTES, TINY_MIN_BYTES, type Rules, type Verdict, type JudgedFile } from './rule_set.js' diff --git a/lib/rules/rule_set.ts b/lib/rules/rule_set.ts index f1e36b4..4c99b29 100644 --- a/lib/rules/rule_set.ts +++ b/lib/rules/rule_set.ts @@ -47,6 +47,12 @@ export interface Rules { * the newznzb.* pieces (9% of the catalog; LOG.md (123), (125)). */ dropNonsensePieces: boolean + /** + * Drop one-article archives of TINY_MIN_BYTES to NONSENSE_MIN_BYTES under a release-like name (spaces allowed), + * outside the exempt groups, without an e-book, subtitle, crack or sidecar word: the 2015-17 exe-stub flood in + * miniature (a RAR holding "<title>.exe" at 2-60 kB; 1.65M catalog rows; LOG.md (131), (133)). + */ + dropTinyArchives: boolean } export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = { @@ -58,6 +64,7 @@ export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = { droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'], dropStubArchives: true, dropNonsensePieces: true, + dropTinyArchives: true, droppedClasses: [], } @@ -69,8 +76,12 @@ export const STUB_WIDE_GROUPS = 4 /** The nonsense-piece rule's bounds: under 100 kB are cue/log sidecars and adverts, 1 MB and up is rule 1's. */ export const NONSENSE_MIN_BYTES = 100_000 export const NONSENSE_MAX_BYTES = 1_000_000 +/** The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars. */ +export const TINY_MIN_BYTES = 2_000 -export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece'; key: string; sample: boolean } +export type Verdict = + | { rule: null } + | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive'; key: string; sample: boolean } /** A file as the judge sees it: its name, parts, groups, poster, and the articles seen with their bytes. */ export interface JudgedFile { @@ -148,6 +159,36 @@ export function isNonsensePiece(f: JudgedFile): boolean { ) } +/** RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way. */ +const RELEASE_LIKE_SPACED = new RegExp(RELEASE_LIKE.source.replace('[._-]', '[._ -]'), 'i') + +/** + * Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of + * exe is the content), covers and sidecars. + */ +const TINY_CONTENT = + /\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b/i + +/** + * A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like, + * unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book, + * subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)). + */ +export function isTinyArchive(f: JudgedFile): boolean { + return ( + f.totalParts === 1 && + f.segments === 1 && + f.bytes >= TINY_MIN_BYTES && + f.bytes < NONSENSE_MIN_BYTES && + !f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g)) && + RELEASE_LIKE_SPACED.test(fileStem(f.name)) && + !EBOOK.test(f.name) && + !TINY_CONTENT.test(f.name) && + !isScrambledFile(f.name) && + !isRandomFile(f.name) + ) +} + export function isRandomFile(name: string) { return looksRandom(fileStem(name)) } @@ -221,6 +262,7 @@ export class RuleSet { if (legs !== '') return { rule: 'stub-archive', key: legs, sample } } if (this.rules.dropNonsensePieces && isNonsensePiece(archive)) return { rule: 'nonsense-piece', key: uploadTool(archive.firstMessageId), sample } + if (this.rules.dropTinyArchives && isTinyArchive(archive)) return { rule: 'tiny-archive', key: uploadTool(archive.firstMessageId), sample } } if (files.some((f) => !isRandomFile(f.name))) return { rule: null } const first = files[0] diff --git a/tests/unit/rules/rule_set.spec.ts b/tests/unit/rules/rule_set.spec.ts index 9579272..16d7aee 100644 --- a/tests/unit/rules/rule_set.spec.ts +++ b/tests/unit/rules/rule_set.spec.ts @@ -98,8 +98,9 @@ test.group('rules / verdicts', () => { test('nonsense pieces: one article of 100 kB to 1 MB under a release-like name', ({ assert }) => { assert.include(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: 'nonsense-piece', key: 'nyuu' }) - // A 3 kB cue/log sidecar rar with a year in its name: under the floor. A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule. - assert.deepEqual(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: null }) + // A 3 kB cue/log sidecar rar with a year in its name: under the floor (the tiny-archive rule's outside the music groups). A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule. + assert.deepEqual(rules.judge('k', [{ ...archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000), groups: ['alt.binaries.sounds.lossless'] }]), { rule: null }) + assert.include(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: 'tiny-archive' }) assert.deepEqual(rules.judge('k', [archive('Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar', 'fake@address.com (Psychopath)', 2, 1, 303_000)]), { rule: null }) assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 382_000)]), { rule: null }) assert.deepEqual(rules.judge('k', [archive('Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar', 'saggir@domain.com', 1, 1, 113_000)]), { rule: null }) @@ -109,6 +110,31 @@ test.group('rules / verdicts', () => { const off = new RuleSet({ ...DEFAULT_RULES, dropNonsensePieces: false, samplePercent: 0, priorities: {}, droppedSignatures: [] }) assert.deepEqual(off.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null }) }) + + test('tiny archives: one article of 2-100 kB under a release name, spaces allowed', ({ assert }) => { + // A RAR holding "<title>.exe": the 2015-17 flood in miniature (LOG.md (131)). + assert.include(rules.judge('k', [archive('Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar', 'Yanc@power-post.org <Yanc@power-post.org>', 1, 1, 15_084)]), { rule: 'tiny-archive', key: 'nyuu' }) + assert.include(rules.judge('k', [archive('Cicero Magazin fuer politische Kultur Mai 05 2016.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 17_478)]), { rule: 'tiny-archive', key: 'nyuu' }) + // A scene name without a year, a tag or a group suffix: not release-like, so kept. + assert.deepEqual(rules.judge('k', [archive('[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar', 'Yanc@power-post.org', 2, 1, 23_449)]), { rule: null }) + assert.include(rules.judge('k', [archive('Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 7_869)]), { rule: 'tiny-archive', key: 'nyuu' }) + // Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives. + assert.deepEqual(rules.judge('k', [archive('The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar', 'henk@example.org', 1, 1, 38_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar', 'poster@example.org', 1, 1, 60_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Some Band - Album (2011) covers.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 38_000)]), { rule: null }) + // A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's; not settled. + assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl (2017).zip', 'home@home.com', 1, 1, 60_000), groups: ['alt.binaries.sounds.karaoke'] }]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 2, 20_000)]), { rule: null }) + assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 1_500)]), { rule: null }) + assert.include(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 150_000)]), { rule: 'nonsense-piece' }) + assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)], false), { rule: null }) + const off = new RuleSet({ ...DEFAULT_RULES, dropTinyArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] }) + assert.deepEqual(off.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null }) + }) }) test.group('rules / upload tools', () => {