From 81a5dce45ac40f0e4275e527cfb224d2fb8a3b04 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 2 Nov 2026 10:01:31 +0100 Subject: [PATCH] Readable names: a token-by-token classifier for the readable flag Searches with the readable filter on returned scrambles: with a random 4-letter query nearly every hit was one ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ", "bd5b...-addon"), and 15% of a 6,000-row sample of the catalog's readable names were. looksRandom knows single-token shapes only; separators, spaces, a random "extension" of any length and a packaging word hid the rest. isUnreadableName (lib/release/readable.ts) and names::is_unreadable_name read a name token by token: words (one case or Capitalized), scrambles (letter/digit and case switches, lone letters between capitals, inner capital runs, few vowels, hex) and neutral tokens (digits, codes like S02E10 and 1080p, CamelCase, packaging words); a name is unreadable when its scrambles outweigh its words. Scene tags' "i" for "I" and the NZBSPLIT hash are read as such. On the evaluation sets: 554 of 559 random-search names caught, 0 of 630 real names flagged (LOG.md (129)). The edge's readable flag and the prober's inner names use it; the drop rules keep looksRandom, which is narrower. An unnamed release goes to the pending store (decision f), so more of the obfuscated groups' releases wait for a probe now. releases:readable clears the flag on the existing catalog's rows by name (a dry run by default). Co-Authored-By: Claude Fable 5.1 --- REBUILD.md | 6 +- commands/releases_readable.ts | 83 ++++++++++++ edge/src/indexer.rs | 4 +- edge/src/names.rs | 188 +++++++++++++++++++++++++++ lib/edge/range_indexer.ts | 3 +- lib/probe/inspect.ts | 4 +- lib/release/index.ts | 1 + lib/release/readable.ts | 136 +++++++++++++++++++ tests/functional/batch_store.spec.ts | 12 +- tests/unit/release/readable.spec.ts | 72 ++++++++++ 10 files changed, 497 insertions(+), 12 deletions(-) create mode 100644 commands/releases_readable.ts create mode 100644 lib/release/readable.ts create mode 100644 tests/unit/release/readable.spec.ts diff --git a/REBUILD.md b/REBUILD.md index c99b733..223b966 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -879,7 +879,11 @@ e. Headers through the coordinator's relay, with prefetch (part 1). f. **Nothing enters the catalog without a usable name** (amended 2026-10-26, the user's direction; before: random-named releases keep no chunk but their rows). A release the window closed on whose files - all have random names is no release yet: it goes to `pending_releases` + all have random names (`isUnreadableName`, `names::is_unreadable_name`: + a name read token by token, so separators, spaces and random + "extensions" don't hide a scramble; since 2026-11-02, LOG.md (129), + when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu" + and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases` (migration 6300, the HDD) with its keys, counts, article span, upload tool and probe targets, no row in `releases`, `release_texts` or `posters`, no chunk. The pending ladder works them **from least to most diff --git a/commands/releases_readable.ts b/commands/releases_readable.ts new file mode 100644 index 0000000..bd23674 --- /dev/null +++ b/commands/releases_readable.ts @@ -0,0 +1,83 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import db from '@adonisjs/lucid/services/db' +import { isUnreadableName } from '#lib/release/index' + +/** Its place in worker_cursors (the last release id done), for a run that was stopped. */ +const CURSOR = 'readable' + +/** + * The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged + * readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable = + * false, so the readable filter hides them (an inner name from the prober still shows them). By the release name, + * which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on. + * Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again. + */ +export default class ReleasesReadable extends BaseCommand { + static commandName = 'releases:readable' + static description = + 'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply' + static options: CommandOptions = { startApp: true } + + @flags.number({ description: 'Release ids per batch' }) + declare batch: number + + @flags.number({ description: 'Stop after this id (a sample)' }) + declare toId: number + + @flags.boolean({ description: 'Write the flag; without it, count and show a sample' }) + declare apply: boolean + + @flags.boolean({ description: 'Start from the first id, not the cursor' }) + declare restart: boolean + + async run() { + const batch = this.batch ?? 20_000 + const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`) + const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity) + const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [ + CURSOR, + ]) + let after = + this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key) + let looked = 0 + let changed = 0 + const sample: string[] = [] + const started = Date.now() + while (after < hi) { + const to = Math.min(after + batch, hi) + const { rows } = await db.rawQuery( + `SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`, + [after, to] + ) + looked += rows.length + const ids: number[] = [] + for (const r of rows) { + if (!isUnreadableName(r.name)) continue + ids.push(Number(r.id)) + if (sample.length < 40) sample.push(r.name) + } + changed += ids.length + if (this.apply) { + if (ids.length > 0) + await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [ + ids, + ]) + await db.rawQuery( + `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`, + [CURSOR, String(to)] + ) + } + after = to + if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) { + this.logger.info( + `Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).` + ) + } + } + this.logger.info( + `${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.` + ) + if (!this.apply) for (const n of sample) this.logger.info(` ${n}`) + } +} diff --git a/edge/src/indexer.rs b/edge/src/indexer.rs index 5c54515..1ff2cba 100644 --- a/edge/src/indexer.rs +++ b/edge/src/indexer.rs @@ -5,7 +5,7 @@ use crate::codec::{encode_chunk, ChunkFile}; use crate::js::{self, sha1_hex}; -use crate::names::{self, is_random_file, looks_scrambled, JudgedFile, Release, Rules, Verdict}; +use crate::names::{self, is_random_file, is_unreadable_name, looks_scrambled, JudgedFile, Release, Rules, Verdict}; use rustc_hash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; use std::rc::Rc; @@ -1560,7 +1560,7 @@ fn chunk_of(key: &str, files: &[Vec], upload: usize) -> ReleaseChunk { bytes, first_article, last_article, - readable: chunk_files.iter().any(|f| !is_random_file(f.name)), + readable: chunk_files.iter().any(|f| !is_unreadable_name(f.name)), has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)), finished: true, edge: false, diff --git a/edge/src/names.rs b/edge/src/names.rs index 81583e4..9ffa9be 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -1065,6 +1065,169 @@ pub fn is_random_file(name: &str) -> bool { looks_random(&file_stem(name)) } +/// Readable or not (lib/release/readable.ts): whether a name says what a release is, read token by token, so +/// separators, spaces and random "extensions" don't hide a scramble that looks_random's single-token shapes miss +/// (LOG.md (129)). The readable flag; the drop rules keep looks_random, which is narrower. +/// Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed words. +static NOISE_TOKEN: LazyLock = LazyLock::new(|| re(r"(?i)^(addon|samples?|proof|subs?|nfo|part[0-9]+|vol[0-9]+|description|repost|rep|extras?|bonus|cd[0-9]*|dvd[0-9]*|dis[ck][0-9]*|par2|rar|zip|7z|mkv|mp4|avi|vol)$")); +/// Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th. +static CODE_TOKEN: LazyLock = LazyLock::new(|| re(r"^([sS][0-9]{1,2}[eE][0-9]{1,3}|[xh]26[45]|[A-Za-z]+[0-9]+([a-z]{1,3}|x(64|86))?|[0-9]+[A-Za-z]{1,3}|[0-9]+[A-Za-z]+[0-9]+)$")); +/// The NZBSPLIT tools' hash between their markers is no part of the name. +static NZBSPLIT_HASH: LazyLock = LazyLock::new(|| re(r"(?i)__NZBS[P]LIT__[0-9a-f]{32}__NZBS[P]LIT__")); // "SP" is re()'s space class +/// An inner run of 2+ capitals with lowercase on both sides ("tBWTuz", "YDoaROv"; not "3DMark03"). +static INNER_UPPER: LazyLock = LazyLock::new(|| re(r"[a-z][A-Z]{2,}[a-z]")); + +/// Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there. +fn scene_case(t: &[u8]) -> Vec { + let mut out = t.to_vec(); + for i in 1..t.len().saturating_sub(1) { + if t[i] == b'i' && t[i - 1].is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() { + out[i] = b'I'; + } + } + out +} + +/// The signals of a scrambled token (readable.ts signals()). +struct TokenSignals { + /// Letter/digit and lower-to-upper switches. + switches: usize, + /// Case switches in both directions. + case_switches: usize, + /// Lone lowercase letters (not "i") between capitals ("NtpUgCmS"). + lone: usize, + inner_upper: bool, + max_consonants: usize, + /// Vowels, a lowercase y too ("McCartney"). + vowels: usize, + /// Runs of one character class. + runs: usize, + letters: usize, + /// The longest lowercase run. + max_lower: usize, +} + +fn token_signals(t: &[u8]) -> TokenSignals { + let is_vowel = |c: u8| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u') || c == b'y'; + let class = |c: u8| if c.is_ascii_digit() { 0 } else if c.is_ascii_lowercase() { 1 } else { 2 }; + let mut s = TokenSignals { switches: 0, case_switches: 0, lone: 0, inner_upper: false, max_consonants: 0, vowels: 0, runs: 0, letters: 0, max_lower: 0 }; + let mut consonants = 0; + let mut lower = 0; + for i in 0..t.len() { + let c = t[i]; + if c.is_ascii_alphabetic() && !is_vowel(c) { + consonants += 1; + s.max_consonants = s.max_consonants.max(consonants); + } else { + consonants = 0; + } + if is_vowel(c) { + s.vowels += 1; + } + if c.is_ascii_alphabetic() { + s.letters += 1; + } + if c.is_ascii_lowercase() { + lower += 1; + s.max_lower = s.max_lower.max(lower); + } else { + lower = 0; + } + if i == 0 || class(t[i - 1]) != class(c) { + s.runs += 1; + } + if i == 0 { + continue; + } + let a = t[i - 1]; + if a.is_ascii_digit() != c.is_ascii_digit() { + s.switches += 1; + } else if a.is_ascii_lowercase() && c.is_ascii_uppercase() { + s.switches += 1; + } + if (a.is_ascii_lowercase() && c.is_ascii_uppercase()) || (a.is_ascii_uppercase() && c.is_ascii_lowercase()) { + s.case_switches += 1; + } + if i < t.len() - 1 && c.is_ascii_lowercase() && c != b'i' && a.is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() { + s.lone += 1; + } + } + s.inner_upper = INNER_UPPER.is_match(t); + s +} + +#[derive(PartialEq, Eq, Debug, Clone, Copy)] +pub enum TokenKind { + Word, + Random, + Neutral, +} + +/// What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a +/// packaging word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with readable.ts. +pub fn token_kind(t: &[u8]) -> TokenKind { + let n = t.len(); + if t.iter().all(u8::is_ascii_digit) || NOISE_TOKEN.is_match(t) || CODE_TOKEN.is_match(t) { + return TokenKind::Neutral; + } + if t.iter().all(u8::is_ascii_alphabetic) { + let s = token_signals(&scene_case(t)); + let capitalized = t[0].is_ascii_uppercase() && t[1..].iter().all(u8::is_ascii_lowercase); + if s.case_switches == 0 || (s.case_switches == 1 && capitalized) { + // A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word. + if n >= 10 && s.vowels * 6 < n { + return TokenKind::Random; + } + return if n >= 3 { TokenKind::Word } else { TokenKind::Neutral }; + } + // Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs. + if n >= 6 && s.case_switches >= 2 && (s.inner_upper || (s.vowels * 4 < n && (s.runs >= 4 || s.lone >= 1))) { + return TokenKind::Random; + } + return TokenKind::Neutral; + } + if n >= 8 && t.iter().all(u8::is_ascii_hexdigit) && t.iter().filter(|c| c.is_ascii_digit()).count() >= 2 { + return TokenKind::Random; + } + if t.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) { + // 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay. + return if n >= 12 && token_signals(t).switches >= 6 { TokenKind::Random } else { TokenKind::Neutral }; + } + let s = token_signals(&scene_case(t)); + if n >= 5 && s.switches >= 3 { + return TokenKind::Random; + } + if n >= 6 && s.switches >= 2 && (s.lone >= 1 || s.inner_upper || s.max_consonants >= 4) { + return TokenKind::Random; + } + // Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus"). + if n >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.max_lower <= 2 { + return TokenKind::Random; + } + TokenKind::Neutral +} + +/// A file or release name no one can read (readable.ts isUnreadableName): looks_random's shapes, or more scrambled +/// tokens than words by letter count ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not +/// "Paul McCartney - Band on the Run", "ShesSoWild - Ester Ana 1080p"). +pub fn is_unreadable_name(name: &str) -> bool { + let stem = file_stem(name); + let stem = NZBSPLIT_HASH.replace(stem.as_bytes(), &b" "[..]); + if looks_random(&String::from_utf8_lossy(&stem)) { + return true; + } + let mut words = 0; + let mut random = 0; + for t in stem.split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) { + match token_kind(t) { + TokenKind::Word => words += t.len(), + TokenKind::Random => random += t.len(), + TokenKind::Neutral => {} + } + } + random > 0 && random > words +} + /// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case /// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"). /// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators. @@ -1622,6 +1785,31 @@ mod tests { assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key); } + #[test] + fn unreadable_names() { + use super::is_unreadable_name; + // Scrambles with separators, spaces, a random "extension", a packaging word, a hex with an addon (LOG.md (129)). + for n in [ + "Zx5AjNaK-mh7CFg6Hf8uu.part01.rar", "yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU", "PsNdnYiugfZ.rar", "phfplC9y07U2yrj", "n5d1ueIf34X", + "tJqr1tq7xK", "Description - Lt5nHjlo3RiuBb3C.par2", "bd5b4a59f3aa87114dcf75c0-addon.rar", "mrrtmzrzmi4dgmrzmyytqmlbztczbx", + "AA-NaR16302830434zxP32108309f3Ab", "_QBs01_MqZYnYwM_iAyRquhc-", "sgkra vU7u2yuw9LDLCjDu04pOez", "Jk5Pud.zQdNj", "7uKsD_NRgT-G_Uc_p8", + "NtpUgCmS_Xa1B2Pk", "{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}", "IkYovjrtZrItjQ.mkv", "ff75ce3f16c746c18b2d78cf.6c52f", + ] { + assert!(is_unreadable_name(n), "{n}"); + } + // Real names: scene tags, CamelCase, codes, leet, an acronym, a hash beside words. + for n in [ + "Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune", "Paul McCartney - Band on the Run", + "ShesSoWild - Ester Ana 1080p", "MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS", "cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar", + "Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing", "Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON", + "Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr", "Eclipse_IDE_2023-12_R_win32_x86_64.zip", "PartsHPGL 1.06c", "Go4Customer Fax Machine v1.2b", + "AdBeGone 1.2", "3DMark03 320", "COD4MW2REMASTERED", "Movie.2020.1080p-GRP [a3f9e2b1]", "EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps", + "iPhone", "Some.Show.S01E01.1080p.mkv", + ] { + assert!(!is_unreadable_name(n), "{n}"); + } + } + #[test] fn random_names() { assert!(looks_random("prvjr7hapd0i1ny14pz4")); diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 607721f..59a2cfc 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -2,6 +2,7 @@ import { hash } from 'node:crypto' import type { Overview } from '../nntp/index.js' import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js' import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js' +import { isUnreadableName } from '../release/index.js' import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js' import { encodeChunk, type ChunkFile } from '../chunks/index.js' @@ -458,7 +459,7 @@ export function chunkOf(key: string, files: AggregatedFile[], uploa bytes, firstArticle, lastArticle, - readable: chunkFiles.some((f) => !isRandomFile(f.name)), + readable: chunkFiles.some((f) => !isUnreadableName(f.name)), hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)), partial: files.some((f) => f.segments.length < f.segments[0].totalParts), payload: encodeChunk(chunkFiles), diff --git a/lib/probe/inspect.ts b/lib/probe/inspect.ts index 281cff2..69f6811 100644 --- a/lib/probe/inspect.ts +++ b/lib/probe/inspect.ts @@ -5,7 +5,7 @@ import { parseRarHeaders } from '#lib/rar/index' import { parseSevenZipTail } from '#lib/sevenzip/index' import { parseZipStart } from '#lib/zip/index' import { sniffContentType, type ContentType } from '#lib/content/index' -import { fileStem, looksRandom } from '#lib/release/index' +import { isUnreadableName } from '#lib/release/index' /** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */ export interface Inspection { @@ -139,7 +139,7 @@ export function usableName(inspection: Inspection, kind: string): string | null const usable = (n: string | undefined) => { if (!n) return null const name = n.replace(/^.*[\\/]/, '').slice(0, 255) - return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null + return name && !PACKAGING.test(name) && !isUnreadableName(name) ? name : null } const named = (inspection.names ?? []).map(usable).find((n) => n) if (named) return named diff --git a/lib/release/index.ts b/lib/release/index.ts index f6eaf78..23fdb54 100644 --- a/lib/release/index.ts +++ b/lib/release/index.ts @@ -3,3 +3,4 @@ */ export { releaseIdentity, displayName, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js' export { looksRandom } from './random.js' +export { isUnreadableName, tokenKind } from './readable.js' diff --git a/lib/release/readable.ts b/lib/release/readable.ts new file mode 100644 index 0000000..e193d7f --- /dev/null +++ b/lib/release/readable.ts @@ -0,0 +1,136 @@ +import { fileStem } from './identity.js' +import { looksRandom } from './random.js' + +/** + * Readable or not: whether a name says what a release is (the catalog's readable flag, the prober's inner names). + * looksRandom (random.ts) knows the single-token shapes; this reads a name token by token, so separators, spaces + * and random "extensions" don't hide a scramble (LOG.md (129)). + */ +/** + * Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed + * words ("Description - Lt5nHjlo3RiuBb3C", "bd5b4a59f3aa87114dcf75c0-addon"). + */ +const NOISE = + /^(addon|samples?|proof|subs?|nfo|part\d+|vol\d+|description|repost|rep|extras?|bonus|cd\d*|dvd\d*|dis[ck]\d*|par2|rar|zip|7z|mkv|mp4|avi|vol)$/i +/** Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th. */ +const CODE = + /^([sS]\d{1,2}[eE]\d{1,3}|[xh]26[45]|[A-Za-z]+\d+([a-z]{1,3}|x(64|86))?|\d+[A-Za-z]{1,3}|\d+[A-Za-z]+\d+)$/ +/** The NZBSPLIT tools' hash between their markers is no part of the name. */ +const NZBSPLIT = /__NZBSPLIT__[0-9a-f]{32}__NZBSPLIT__/i + +/** Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there. */ +function sceneCase(t: string): string { + let out = '' + for (let i = 0; i < t.length; i++) { + out += + t[i] === 'i' && i > 0 && i < t.length - 1 && /[A-Z]/.test(t[i - 1]) && /[A-Z]/.test(t[i + 1]) + ? 'I' + : t[i] + } + return out +} + +/** + * The signals of a scrambled token: letter/digit and lower-to-upper switches; case switches in both directions; lone + * lowercase letters between capitals ("NtpUgCmS"); an inner run of 2+ capitals with lowercase on both sides + * ("tBWTuz", "YDoaROv"); the longest consonant run; vowels (a lowercase y too: "McCartney"); runs of one character class. + */ +function signals(t: string) { + let switches = 0 + let caseSwitches = 0 + let lone = 0 + let consonants = 0 + let maxConsonants = 0 + for (let i = 0; i < t.length; i++) { + const c = t[i] + if (/[b-df-hj-np-tv-z]/i.test(c)) maxConsonants = Math.max(maxConsonants, ++consonants) + else consonants = 0 + if (i === 0) continue + const a = t[i - 1] + if (/\d/.test(a) !== /\d/.test(c)) switches++ + else if (/[a-z]/.test(a) && /[A-Z]/.test(c)) switches++ + if ((/[a-z]/.test(a) && /[A-Z]/.test(c)) || (/[A-Z]/.test(a) && /[a-z]/.test(c))) caseSwitches++ + if ( + i < t.length - 1 && + /[a-z]/.test(c) && + c !== 'i' && + /[A-Z]/.test(a) && + /[A-Z]/.test(t[i + 1]) + ) + lone++ + } + const innerUpper = /[a-z][A-Z]{2,}[a-z]/.test(t) + const vowels = (t.match(/[aeiouAEIOU]|y/g) ?? []).length + const runs = (t.match(/[a-z]+|[A-Z]+|\d+/g) ?? []).length + const letters = (t.match(/[A-Za-z]/g) ?? []).length + const maxLower = Math.max(0, ...(t.match(/[a-z]+/g) ?? []).map((r) => r.length)) + return { + switches, + caseSwitches, + lone, + innerUpper, + maxConsonants, + vowels, + runs, + letters, + maxLower, + } +} + +/** + * What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a packaging + * word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with names.rs token_kind. + */ +export function tokenKind(t: string): 'word' | 'random' | null { + if (/^\d+$/.test(t) || NOISE.test(t) || CODE.test(t)) return null + if (/^[A-Za-z]+$/.test(t)) { + const s = signals(sceneCase(t)) + if (s.caseSwitches === 0 || (s.caseSwitches === 1 && /^[A-Z][a-z]+$/.test(t))) { + // A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word. + if (t.length >= 10 && s.vowels * 6 < t.length) return 'random' + return t.length >= 3 ? 'word' : null + } + // Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs. + if ( + t.length >= 6 && + s.caseSwitches >= 2 && + (s.innerUpper || (s.vowels * 4 < t.length && (s.runs >= 4 || s.lone >= 1))) + ) + return 'random' + return null + } + if (t.length >= 8 && /^[0-9a-f]+$/i.test(t) && (t.match(/\d/g)?.length ?? 0) >= 2) return 'random' + if (/^[A-Z0-9]+$/.test(t)) { + // 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay. + return t.length >= 12 && signals(t).switches >= 6 ? 'random' : null + } + const s = signals(sceneCase(t)) + if (t.length >= 5 && s.switches >= 3) return 'random' + if (t.length >= 6 && s.switches >= 2 && (s.lone >= 1 || s.innerUpper || s.maxConsonants >= 4)) + return 'random' + // Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus"). + if (t.length >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.maxLower <= 2) + return 'random' + return null +} + +/** + * A file or release name no one can read: looksRandom's shapes, or more scrambled tokens than words by letter count + * ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not "Paul McCartney - Band on the Run", + * "ShesSoWild - Ester Ana 1080p"). Separators and a random "extension" of any length don't hide a scramble, which + * looksRandom's single-token shapes miss (15% of the readable catalog on 2026-11-02; LOG.md (129)). The readable + * flag; the drop rules keep looksRandom, which is narrower. + */ +export function isUnreadableName(name: string): boolean { + const stem = fileStem(name).replace(NZBSPLIT, ' ') + if (looksRandom(stem)) return true + let words = 0 + let random = 0 + for (const t of stem.split(/[^A-Za-z0-9]+/)) { + if (!t) continue + const kind = tokenKind(t) + if (kind === 'word') words += t.length + else if (kind === 'random') random += t.length + } + return random > 0 && random > words +} diff --git a/tests/functional/batch_store.spec.ts b/tests/functional/batch_store.spec.ts index 0d7313c..7091994 100644 --- a/tests/functional/batch_store.spec.ts +++ b/tests/functional/batch_store.spec.ts @@ -431,10 +431,10 @@ test.group('BatchStore', (group) => { assert, }) => { await storeIn(GROUP, 1, [ - ...upload(10, 'TsIgNBewuFbuwWy.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'), - ...upload(20, 'TsIgNBewuFbuwWy.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'), - ...upload(30, 'TsIgNBewuFbuwWy.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'), - ...upload(40, 'TsIgNBewuFbuwWy.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'), + ...upload(10, 'Some.Show.S01E04.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'), + ...upload(20, 'Some.Show.S01E04.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'), + ...upload(30, 'Some.Show.S01E04.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'), + ...upload(40, 'Some.Show.S01E04.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'), ...upload( 50, 'Some.Show.S01E03.part01.rar', @@ -455,8 +455,8 @@ test.group('BatchStore', (group) => { candidates.map((c) => [c.kind, c.file]), [ ['archive', 'Some.Show.S01E03.part01.rar'], - ['archive', 'TsIgNBewuFbuwWy.7z.001'], - ['7z-tail', 'TsIgNBewuFbuwWy.7z.004'], + ['archive', 'Some.Show.S01E04.7z.001'], + ['7z-tail', 'Some.Show.S01E04.7z.004'], ] ) }) diff --git a/tests/unit/release/readable.spec.ts b/tests/unit/release/readable.spec.ts new file mode 100644 index 0000000..90fc525 --- /dev/null +++ b/tests/unit/release/readable.spec.ts @@ -0,0 +1,72 @@ +import { test } from '@japa/runner' +import { isUnreadableName, tokenKind } from '#lib/release/index' + +test.group('release / isUnreadableName', () => { + test('scrambles with separators, spaces, random extensions, packaging words and hex with an addon are unreadable', ({ + assert, + }) => { + for (const n of [ + 'Zx5AjNaK-mh7CFg6Hf8uu.part01.rar', + 'yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU', + 'PsNdnYiugfZ.rar', + 'phfplC9y07U2yrj', + 'n5d1ueIf34X', + 'tJqr1tq7xK', + 'Description - Lt5nHjlo3RiuBb3C.par2', + 'bd5b4a59f3aa87114dcf75c0-addon.rar', + 'mrrtmzrzmi4dgmrzmyytqmlbztczbx', + 'AA-NaR16302830434zxP32108309f3Ab', + '_QBs01_MqZYnYwM_iAyRquhc-', + 'sgkra vU7u2yuw9LDLCjDu04pOez', + 'Jk5Pud.zQdNj', + '7uKsD_NRgT-G_Uc_p8', + 'NtpUgCmS_Xa1B2Pk', + '{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}', + 'IkYovjrtZrItjQ.mkv', + 'ff75ce3f16c746c18b2d78cf.6c52f', + 'prvjr7hapd0i1ny14pz4', + ]) { + assert.isTrue(isUnreadableName(n), n) + } + }) + + test('scene tags, CamelCase, codes, leet, acronyms and a hash beside words are readable', ({ + assert, + }) => { + for (const n of [ + 'Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune', + 'Paul McCartney - Band on the Run', + 'ShesSoWild - Ester Ana 1080p', + 'MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS', + 'cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar', + 'Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing', + 'Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON', + 'Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr', + 'Eclipse_IDE_2023-12_R_win32_x86_64.zip', + 'PartsHPGL 1.06c', + 'Go4Customer Fax Machine v1.2b', + 'AdBeGone 1.2', + '3DMark03 320', + 'COD4MW2REMASTERED', + 'Movie.2020.1080p-GRP [a3f9e2b1]', + 'EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps', + 'iPhone', + 'Some.Show.S01E01.1080p.mkv', + ]) { + assert.isFalse(isUnreadableName(n), n) + } + }) + + test('token kinds', ({ assert }) => { + assert.equal(tokenKind('Champions'), 'word') + assert.equal(tokenKind('PRESCRiPTiON'), 'word') + assert.equal(tokenKind('GarciaLive'), null) + assert.equal(tokenKind('S02E10'), null) + assert.equal(tokenKind('1080p'), null) + assert.equal(tokenKind('addon'), null) + assert.equal(tokenKind('NtpUgCmS'), 'random') + assert.equal(tokenKind('tBWTuz'), 'random') + assert.equal(tokenKind('Lt5nHjlo3RiuBb3C'), 'random') + assert.equal(tokenKind('a3f9e2b1'), 'random') + }) +})