diff --git a/REBUILD.md b/REBUILD.md
index 7eab73d..02e2fff 100644
--- a/REBUILD.md
+++ b/REBUILD.md
@@ -806,9 +806,17 @@ unscrambled name without spaces and without an e-book format in it is
dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool):
the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of
the catalog; under 100 kB are cue/log sidecars and adverts, and the names
-with spaces are books and albums. Both rules are switched by the
-rule set (`dropStubArchives`, `dropNonsensePieces`), sampled like every
-drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer
+with spaces are books and albums. Below that, a settled single archive of
+one article and 2–100 kB under a release-like name (spaces allowed), outside
+the exempt groups and without an e-book, subtitle, crack/keygen/trainer,
+cover or sidecar word, is dropped as a **tiny archive** (`tiny-archive`,
+key: the upload tool; LOG.md (131), (133)): the 2015–17 flood's RARs
+holding "
.exe" at 2–60 kB, 1.66M catalog rows from a dozen posters
+(12 of 12 fetched fake); the word exemptions keep the real tiny archives
+(subtitles, cracks, trainers) at the price of ~50k fakes named "Incl
+Keygen" that the prober judges later. The rules are switched by the
+rule set (`dropStubArchives`, `dropNonsensePieces`, `dropTinyArchives`),
+sampled like every drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer
judges as each batch completes files, so it passes settled without the
window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now),
`check.ts` and the fixture comparison (the parity with the TypeScript
diff --git a/app/services/pending_ladder.ts b/app/services/pending_ladder.ts
index 5328dfe..2ef22d2 100644
--- a/app/services/pending_ladder.ts
+++ b/app/services/pending_ladder.ts
@@ -375,21 +375,28 @@ class PendingLadder {
)
}
- /** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
+ /**
+ * Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
+ * many. Driven from the dropped classes and signatures into the claim's index and the ladder's: a scan of the
+ * waiting rows testing each (44M on 2026-11-02) ran 9+ minutes on the HDD every sweep (LOG.md (133)).
+ */
async discard(limit = 10_000): Promise {
const { samplePercent } = await RuleSets.current()
- const { rows } = await db.rawQuery(
- `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
- WHERE p.status = ? AND (EXISTS (
- SELECT 1 FROM pending_signatures s
- WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
- OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
- LIMIT ?`,
+ const { rows: byClass } = await db.rawQuery(
+ `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_classes c
+ JOIN pending_releases p ON p.status = ? AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass
+ WHERE c.dropped LIMIT ?`,
[PENDING.waiting, limit]
)
- const keys = rows
- .map((r: any) => r.release_key as string)
- .filter((key: string) => !explored(key, samplePercent))
+ const { rows: bySignature } = await db.rawQuery(
+ `SELECT encode(p.release_key, 'hex') AS release_key FROM pending_signatures s
+ JOIN pending_releases p ON p.status = ? AND p.group_id = s.group_id AND p.tool = s.tool AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))
+ WHERE s.dropped LIMIT ?`,
+ [PENDING.waiting, limit]
+ )
+ const keys = [
+ ...new Set([...byClass, ...bySignature].map((r: any) => r.release_key as string)),
+ ].filter((key: string) => !explored(key, samplePercent))
if (keys.length === 0) return 0
const { rows: gone } = await db.rawQuery(
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
diff --git a/app/services/pending_namer.ts b/app/services/pending_namer.ts
index 6c8c126..e5a994c 100644
--- a/app/services/pending_namer.ts
+++ b/app/services/pending_namer.ts
@@ -30,6 +30,13 @@ interface NamedRow {
/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
const MAX_MOVES = 3
+/**
+ * The widest article span the namer reads again (a busy group posts 50M articles a day, a large upload among
+ * them spans a million or two): one wider is a name the edge saw again much later ("v-lgpc.3ds"), and its span
+ * of segments would not fit the indexer's memory (LOG.md (133)).
+ */
+const MAX_SPAN = 3_000_000
+
/**
* The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
* read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
@@ -94,6 +101,10 @@ export class PendingNamer {
/** One named release into the catalog. */
async move(row: NamedRow): Promise {
+ if (row.last_article - row.first_article > MAX_SPAN)
+ throw new Error(
+ `its articles ${row.first_article}–${row.last_article} span more than ${MAX_SPAN}`
+ )
const group = await db.from('groups').where('id', row.group_id).firstOrFail()
const range = await db
.from('ranges')
@@ -103,19 +114,20 @@ export class PendingNamer {
.orderBy('id')
.first()
if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
- const text = await this.overviewText(group.name, row)
const rules = await RuleSets.current()
const ix = new RangeIndexer({
group: group.name,
start: row.first_article,
end: row.last_article,
rules: { ...rules, samplePercent: 0 },
- idleChunks: 0,
+ // No file given up before the end: the span is read chunk by chunk (one text of it topped V8's string).
+ idleChunks: Number.MAX_SAFE_INTEGER,
edgeArticles: 0,
// The edge wrote it partial when its file is short of parts on the server: so must the re-read.
partialFiles: true,
})
- ix.addText(text, row.last_article + 1)
+ for await (const piece of this.overviewPieces(group.name, row))
+ ix.addText(piece.text, piece.next)
ix.finish()
const batch = ix.take()
const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
@@ -160,50 +172,20 @@ export class PendingNamer {
})
}
- /** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
- private async overviewText(groupName: string, row: NamedRow): Promise {
- if (this.corpusDir) {
- const stored = await db
- .from('ranges')
- .where('group_id', row.group_id)
- .whereNotNull('stored_at')
- .where('start_id', '<=', row.last_article)
- .where('end_id', '>=', row.first_article)
- .orderBy('start_id')
- const parts: string[] = []
- let covered = true
- for (const r of stored) {
- const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
- const size = coordinatorConfig.chunkArticles
- for (
- let from =
- range.start +
- Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
- from <= Math.min(row.last_article, range.end);
- from += size
- ) {
- const raw = await Corpus.read(
- range,
- { from, to: Math.min(from + size - 1, range.end) },
- this.corpusDir
- )
- if (!raw) {
- covered = false
- break
- }
- parts.push((await inflate(raw)).toString('latin1'))
- }
- if (!covered) break
- }
- if (
- covered &&
- parts.length > 0 &&
- stored.length > 0 &&
- Number(stored[0].start_id) <= row.first_article &&
- Number(stored[stored.length - 1].end_id) >= row.last_article
- ) {
- return parts.join('')
- }
+ /**
+ * The overview lines of the release's article span, a chunk of articles at a time with the first article after
+ * it: the stored corpus chunks when its ranges are stored and contiguous, else XOVER in windows of a chunk.
+ */
+ private async *overviewPieces(
+ groupName: string,
+ row: NamedRow
+ ): AsyncGenerator<{ text: string; next: number }> {
+ const size = coordinatorConfig.chunkArticles
+ const stored = await this.storedChunks(groupName, row)
+ if (stored) {
+ for (const { raw, next } of stored)
+ yield { text: (await inflate(raw)).toString('latin1'), next }
+ return
}
this.client ??= await XoverClient.connect({
host: env.get('NNTP_HOST')!,
@@ -217,13 +199,55 @@ export class PendingNamer {
this.usedAt = Date.now()
try {
await this.client.group(groupName)
- // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
- const raw = await this.client.xoverRaw(row.first_article, row.last_article)
- return (await inflate(raw)).toString('latin1')
+ for (let from = row.first_article; from <= row.last_article; from += size) {
+ const to = Math.min(from + size - 1, row.last_article)
+ // One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
+ const raw = await this.client.xoverRaw(from, to)
+ yield { text: (await inflate(raw)).toString('latin1'), next: to + 1 }
+ }
} catch (error) {
this.client.close()
this.client = null
throw error
}
}
+
+ /** The span's stored corpus chunks (compressed, with the first article after each), or null when any is missing. */
+ private async storedChunks(
+ groupName: string,
+ row: NamedRow
+ ): Promise<{ raw: Buffer; next: number }[] | null> {
+ if (!this.corpusDir) return null
+ const stored = await db
+ .from('ranges')
+ .where('group_id', row.group_id)
+ .whereNotNull('stored_at')
+ .where('start_id', '<=', row.last_article)
+ .where('end_id', '>=', row.first_article)
+ .orderBy('start_id')
+ if (
+ stored.length === 0 ||
+ Number(stored[0].start_id) > row.first_article ||
+ Number(stored[stored.length - 1].end_id) < row.last_article
+ )
+ return null
+ const size = coordinatorConfig.chunkArticles
+ const chunks: { raw: Buffer; next: number }[] = []
+ for (const r of stored) {
+ const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
+ for (
+ let from =
+ range.start +
+ Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
+ from <= Math.min(row.last_article, range.end);
+ from += size
+ ) {
+ const to = Math.min(from + size - 1, range.end)
+ const raw = await Corpus.read(range, { from, to }, this.corpusDir)
+ if (!raw) return null
+ chunks.push({ raw, next: to + 1 })
+ }
+ }
+ return chunks
+ }
}
diff --git a/database/migrations/1770000007000_probe_candidates_taken_index.ts b/database/migrations/1770000007000_probe_candidates_taken_index.ts
new file mode 100644
index 0000000..5d34469
--- /dev/null
+++ b/database/migrations/1770000007000_probe_candidates_taken_index.ts
@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The prober's start puts a previous run's claims back (probe_candidates status taken → waiting), which scanned
+ * the 7.8 GB candidates table on the HDD for 4 minutes after every deploy (LOG.md (133)): a partial index on the
+ * few taken rows makes it instant.
+ */
+export default class extends BaseSchema {
+ static disableTransactions = true
+
+ async up() {
+ this.schema.raw(
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS probe_candidates_taken_idx ON probe_candidates (id) WHERE status = 'taken'`
+ )
+ }
+
+ async down() {
+ this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS probe_candidates_taken_idx`)
+ }
+}
diff --git a/edge/src/lease.rs b/edge/src/lease.rs
index 44e0adb..07278fa 100644
--- a/edge/src/lease.rs
+++ b/edge/src/lease.rs
@@ -77,6 +77,8 @@ pub struct RulesJson {
pub drop_stub_archives: Option,
#[serde(default)]
pub drop_nonsense_pieces: Option,
+ #[serde(default)]
+ pub drop_tiny_archives: Option,
}
impl RulesJson {
@@ -107,6 +109,9 @@ impl RulesJson {
if let Some(d) = self.drop_nonsense_pieces {
rules.drop_nonsense_pieces = d;
}
+ if let Some(d) = self.drop_tiny_archives {
+ rules.drop_tiny_archives = d;
+ }
rules
}
}
diff --git a/edge/src/names.rs b/edge/src/names.rs
index 70a26bd..5255448 100644
--- a/edge/src/names.rs
+++ b/edge/src/names.rs
@@ -1515,6 +1515,11 @@ static RELEASE_LIKE: LazyLock = LazyLock::new(|| {
re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
});
+/// RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way.
+static RELEASE_LIKE_SPACED: LazyLock = LazyLock::new(|| {
+ re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)")
+});
+
/// The stub-archive rule (LOG.md (123)-(126), (128)): one unsplit archive, no par2, settled, posted to 2+ groups,
/// under STUB_MAX_BYTES (the fakes stop at ~90 MB; the hash-named uploads, real albums and passworded videos above
/// it don't), and its articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's
@@ -1528,6 +1533,8 @@ pub const STUB_WIDE_GROUPS: usize = 4;
/// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts).
pub const NONSENSE_MIN_BYTES: u64 = 100_000;
pub const NONSENSE_MAX_BYTES: u64 = 1_000_000;
+/// The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars.
+pub const TINY_MIN_BYTES: u64 = 2_000;
pub struct JudgedFile<'a> {
pub name: &'a str,
@@ -1589,6 +1596,27 @@ pub fn is_nonsense_piece(f: &JudgedFile) -> bool {
&& !is_random_file(f.name)
}
+/// Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of
+/// exe is the content), covers and sidecars.
+static TINY_CONTENT: LazyLock = LazyLock::new(|| {
+ re(r"(?i)\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b")
+});
+
+/// A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like,
+/// unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book,
+/// subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)).
+pub fn is_tiny_archive(f: &JudgedFile) -> bool {
+ f.total_parts == 1
+ && f.segments == 1
+ && (TINY_MIN_BYTES..NONSENSE_MIN_BYTES).contains(&f.bytes)
+ && !f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g))
+ && test(&RELEASE_LIKE_SPACED, &file_stem(f.name))
+ && !test(&EBOOK, f.name)
+ && !test(&TINY_CONTENT, f.name)
+ && !looks_scrambled(f.name)
+ && !is_random_file(f.name)
+}
+
pub enum Verdict {
Keep,
Drop { rule: &'static str, key: String, sample: bool },
@@ -1610,9 +1638,11 @@ pub struct Rules {
pub drop_executables: bool,
/// Fragments of the poster lowercased without punctuation (droppedPosters).
pub dropped_posters: Vec,
- /// The stub-archive rule (dropStubArchives) and the nonsense-piece rule (dropNonsensePieces).
+ /// The stub-archive rule (dropStubArchives), the nonsense-piece rule (dropNonsensePieces) and the tiny-archive
+ /// rule (dropTinyArchives).
pub drop_stub_archives: bool,
pub drop_nonsense_pieces: bool,
+ pub drop_tiny_archives: bool,
}
impl Rules {
@@ -1630,6 +1660,7 @@ impl Rules {
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
drop_stub_archives: true,
drop_nonsense_pieces: true,
+ drop_tiny_archives: true,
}
}
}
@@ -1665,6 +1696,9 @@ pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rul
if rules.drop_nonsense_pieces && is_nonsense_piece(f) {
return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample };
}
+ if rules.drop_tiny_archives && is_tiny_archive(f) {
+ return Verdict::Drop { rule: "tiny-archive", key: upload_tool(f.first_message_id).to_string(), sample };
+ }
}
if files.iter().any(|f| !is_random_file(f.name)) {
return Verdict::Keep;
@@ -1993,8 +2027,8 @@ mod single_archive_rule_tests {
let rules = Rules::defaults(Default::default());
// One 790 kB article under a release-like name: the newznzb.* pieces.
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
- // A 3 kB cue/log sidecar rar with a year in its name, and a diet advert: under the floor.
- assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), None);
+ // A 3 kB cue/log sidecar rar with a year in its name: under this rule's floor (the tiny-archive rule's outside the music groups).
+ assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
// A book (a year or a format in its name, written with spaces), a dotted one too: not a piece.
assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None);
assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None);
@@ -2006,6 +2040,35 @@ mod single_archive_rule_tests {
let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) };
assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None);
}
+
+ #[test]
+ fn tiny_archives() {
+ let rules = Rules::defaults(Default::default());
+ // One article of 2-100 kB under a release name, dotted or with spaces: a RAR holding ".exe" (LOG.md (131)).
+ assert_eq!(verdict("Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar", "Yanc@power-post.org ", 1, 1, 15_084, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
+ assert_eq!(verdict("Cicero Magazin fuer politische Kultur Mai 05 2016.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 17_478, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
+ // A scene name without a year, a tag or a group suffix: not release-like, so kept (the flood's porn names mostly carry one).
+ assert_eq!(verdict("[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar", "Yanc@power-post.org", 2, 1, 23_449, true, &rules), None);
+ assert_eq!(verdict("Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 7_869, true, &rules), Some(("tiny-archive", "nyuu".to_string())));
+ // Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives.
+ assert_eq!(verdict("The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar", "henk@example.org", 1, 1, 38_000, true, &rules), None);
+ assert_eq!(verdict("Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar", "poster@example.org", 1, 1, 60_000, true, &rules), None);
+ assert_eq!(verdict("Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None);
+ assert_eq!(verdict("Some Band - Album (2011) covers.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None);
+ assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 38_000, true, &rules), None);
+ // A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's.
+ let groups = vec!["alt.binaries.sounds.karaoke".to_string()];
+ let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl (2017).zip", total_parts: 1, groups: &groups, first_message_id: NYUU, poster: "home@home.com", segments: 1, bytes: 60_000 }];
+ assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep));
+ assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None);
+ assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None);
+ assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 2, 20_000, true, &rules), None);
+ assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 1_500, true, &rules), None);
+ assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 150_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string())));
+ assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, false, &rules), None);
+ let off = Rules { drop_tiny_archives: false, ..Rules::defaults(Default::default()) };
+ assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, true, &off), None);
+ }
}
#[cfg(test)]
diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts
index 40e6620..b21ffa2 100644
--- a/lib/edge/range_indexer.ts
+++ b/lib/edge/range_indexer.ts
@@ -66,7 +66,7 @@ export interface ProbeCandidate {
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
export interface RuleCount {
- rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'incomplete'
+ rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive' | 'incomplete'
key: string
articles: number
files: number
diff --git a/lib/rules/index.ts b/lib/rules/index.ts
index a292266..5b810e3 100644
--- a/lib/rules/index.ts
+++ b/lib/rules/index.ts
@@ -1,2 +1,2 @@
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
-export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, stubArchiveLegs, isNonsensePiece, STUB_ARTICLE_BYTES, STUB_MIN_BYTES, STUB_MAX_BYTES, STUB_WIDE_GROUPS, NONSENSE_MIN_BYTES, NONSENSE_MAX_BYTES, type Rules, type Verdict, type JudgedFile } from './rule_set.js'
+export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, stubArchiveLegs, isNonsensePiece, isTinyArchive, STUB_ARTICLE_BYTES, STUB_MIN_BYTES, STUB_MAX_BYTES, STUB_WIDE_GROUPS, NONSENSE_MIN_BYTES, NONSENSE_MAX_BYTES, TINY_MIN_BYTES, type Rules, type Verdict, type JudgedFile } from './rule_set.js'
diff --git a/lib/rules/rule_set.ts b/lib/rules/rule_set.ts
index f1e36b4..4c99b29 100644
--- a/lib/rules/rule_set.ts
+++ b/lib/rules/rule_set.ts
@@ -47,6 +47,12 @@ export interface Rules {
* the newznzb.* pieces (9% of the catalog; LOG.md (123), (125)).
*/
dropNonsensePieces: boolean
+ /**
+ * Drop one-article archives of TINY_MIN_BYTES to NONSENSE_MIN_BYTES under a release-like name (spaces allowed),
+ * outside the exempt groups, without an e-book, subtitle, crack or sidecar word: the 2015-17 exe-stub flood in
+ * miniature (a RAR holding ".exe" at 2-60 kB; 1.65M catalog rows; LOG.md (131), (133)).
+ */
+ dropTinyArchives: boolean
}
export const DEFAULT_RULES: Omit = {
@@ -58,6 +64,7 @@ export const DEFAULT_RULES: Omit = {
droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
dropStubArchives: true,
dropNonsensePieces: true,
+ dropTinyArchives: true,
droppedClasses: [],
}
@@ -69,8 +76,12 @@ export const STUB_WIDE_GROUPS = 4
/** The nonsense-piece rule's bounds: under 100 kB are cue/log sidecars and adverts, 1 MB and up is rule 1's. */
export const NONSENSE_MIN_BYTES = 100_000
export const NONSENSE_MAX_BYTES = 1_000_000
+/** The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars. */
+export const TINY_MIN_BYTES = 2_000
-export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece'; key: string; sample: boolean }
+export type Verdict =
+ | { rule: null }
+ | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece' | 'tiny-archive'; key: string; sample: boolean }
/** A file as the judge sees it: its name, parts, groups, poster, and the articles seen with their bytes. */
export interface JudgedFile {
@@ -148,6 +159,36 @@ export function isNonsensePiece(f: JudgedFile): boolean {
)
}
+/** RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way. */
+const RELEASE_LIKE_SPACED = new RegExp(RELEASE_LIKE.source.replace('[._-]', '[._ -]'), 'i')
+
+/**
+ * Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of
+ * exe is the content), covers and sidecars.
+ */
+const TINY_CONTENT =
+ /\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b/i
+
+/**
+ * A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like,
+ * unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book,
+ * subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)).
+ */
+export function isTinyArchive(f: JudgedFile): boolean {
+ return (
+ f.totalParts === 1 &&
+ f.segments === 1 &&
+ f.bytes >= TINY_MIN_BYTES &&
+ f.bytes < NONSENSE_MIN_BYTES &&
+ !f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g)) &&
+ RELEASE_LIKE_SPACED.test(fileStem(f.name)) &&
+ !EBOOK.test(f.name) &&
+ !TINY_CONTENT.test(f.name) &&
+ !isScrambledFile(f.name) &&
+ !isRandomFile(f.name)
+ )
+}
+
export function isRandomFile(name: string) {
return looksRandom(fileStem(name))
}
@@ -221,6 +262,7 @@ export class RuleSet {
if (legs !== '') return { rule: 'stub-archive', key: legs, sample }
}
if (this.rules.dropNonsensePieces && isNonsensePiece(archive)) return { rule: 'nonsense-piece', key: uploadTool(archive.firstMessageId), sample }
+ if (this.rules.dropTinyArchives && isTinyArchive(archive)) return { rule: 'tiny-archive', key: uploadTool(archive.firstMessageId), sample }
}
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
const first = files[0]
diff --git a/tests/unit/rules/rule_set.spec.ts b/tests/unit/rules/rule_set.spec.ts
index 9579272..16d7aee 100644
--- a/tests/unit/rules/rule_set.spec.ts
+++ b/tests/unit/rules/rule_set.spec.ts
@@ -98,8 +98,9 @@ test.group('rules / verdicts', () => {
test('nonsense pieces: one article of 100 kB to 1 MB under a release-like name', ({ assert }) => {
assert.include(rules.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: 'nonsense-piece', key: 'nyuu' })
- // A 3 kB cue/log sidecar rar with a year in its name: under the floor. A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule.
- assert.deepEqual(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: null })
+ // A 3 kB cue/log sidecar rar with a year in its name: under the floor (the tiny-archive rule's outside the music groups). A book (spaces, a format). Two articles, a plain name, a scrambled one: not this rule.
+ assert.deepEqual(rules.judge('k', [{ ...archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000), groups: ['alt.binaries.sounds.lossless'] }]), { rule: null })
+ assert.include(rules.judge('k', [archive('The Smiths-the queen is dead(Remaster)(2011).rar', 'Yenc@power-post.org (Sir Cedrik)', 1, 1, 3_000)]), { rule: 'tiny-archive' })
assert.deepEqual(rules.judge('k', [archive('Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar', 'fake@address.com (Psychopath)', 2, 1, 303_000)]), { rule: null })
assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 382_000)]), { rule: null })
assert.deepEqual(rules.judge('k', [archive('Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar', 'saggir@domain.com', 1, 1, 113_000)]), { rule: null })
@@ -109,6 +110,31 @@ test.group('rules / verdicts', () => {
const off = new RuleSet({ ...DEFAULT_RULES, dropNonsensePieces: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
assert.deepEqual(off.judge('k', [archive('Some.Show.S01E01.1080p.WEB-DL.rar', 'poster@example.org', 1, 1, 790_000)]), { rule: null })
})
+
+ test('tiny archives: one article of 2-100 kB under a release name, spaces allowed', ({ assert }) => {
+ // A RAR holding ".exe": the 2015-17 flood in miniature (LOG.md (131)).
+ assert.include(rules.judge('k', [archive('Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar', 'Yanc@power-post.org ', 1, 1, 15_084)]), { rule: 'tiny-archive', key: 'nyuu' })
+ assert.include(rules.judge('k', [archive('Cicero Magazin fuer politische Kultur Mai 05 2016.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 17_478)]), { rule: 'tiny-archive', key: 'nyuu' })
+ // A scene name without a year, a tag or a group suffix: not release-like, so kept.
+ assert.deepEqual(rules.judge('k', [archive('[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar', 'Yanc@power-post.org', 2, 1, 23_449)]), { rule: null })
+ assert.include(rules.judge('k', [archive('Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar', 'WarezGeek@WarezGeek.org (WarezGeek)', 1, 1, 7_869)]), { rule: 'tiny-archive', key: 'nyuu' })
+ // Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives.
+ assert.deepEqual(rules.judge('k', [archive('The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar', 'henk@example.org', 1, 1, 38_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar', 'poster@example.org', 1, 1, 60_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('Some Band - Album (2011) covers.rar', 'poster@example.org', 1, 1, 45_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar', 'fake@address.com (Psychopath)', 1, 1, 38_000)]), { rule: null })
+ // A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's; not settled.
+ assert.deepEqual(rules.judge('k', [{ ...archive('KV-1055806 - Walker Hayes - U Gurl (2017).zip', 'home@home.com', 1, 1, 60_000), groups: ['alt.binaries.sounds.karaoke'] }]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('holiday snaps.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('nYDeXeF49dieZ1J7flK-JFzZ3j.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 2, 20_000)]), { rule: null })
+ assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 1_500)]), { rule: null })
+ assert.include(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 150_000)]), { rule: 'nonsense-piece' })
+ assert.deepEqual(rules.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)], false), { rule: null })
+ const off = new RuleSet({ ...DEFAULT_RULES, dropTinyArchives: false, samplePercent: 0, priorities: {}, droppedSignatures: [] })
+ assert.deepEqual(off.judge('k', [archive('Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar', 'poster@example.org', 1, 1, 20_000)]), { rule: null })
+ })
})
test.group('rules / upload tools', () => {