Readable names: a token-by-token classifier for the readable flag
Test and deploy / test (push) Successful in 3m36s
Test and deploy / deploy (push) Skipped

Searches with the readable filter on returned scrambles: with a random
4-letter query nearly every hit was one ("Zx5AjNaK-mh7CFg6Hf8uu",
"Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ", "bd5b...-addon"), and
15% of a 6,000-row sample of the catalog's readable names were. looksRandom
knows single-token shapes only; separators, spaces, a random "extension"
of any length and a packaging word hid the rest.

isUnreadableName (lib/release/readable.ts) and names::is_unreadable_name
read a name token by token: words (one case or Capitalized), scrambles
(letter/digit and case switches, lone letters between capitals, inner
capital runs, few vowels, hex) and neutral tokens (digits, codes like
S02E10 and 1080p, CamelCase, packaging words); a name is unreadable when
its scrambles outweigh its words. Scene tags' "i" for "I" and the
NZBSPLIT hash are read as such. On the evaluation sets: 554 of 559
random-search names caught, 0 of 630 real names flagged (LOG.md (129)).

The edge's readable flag and the prober's inner names use it; the drop
rules keep looksRandom, which is narrower. An unnamed release goes to the
pending store (decision f), so more of the obfuscated groups' releases
wait for a probe now. releases:readable clears the flag on the existing
catalog's rows by name (a dry run by default).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-11-02 10:01:31 +01:00
1 parent 2643266559
commit 81a5dce45a
10 files changed
+497 -12

No files matched your search

+5 -1
View File
@@ -879,7 +879,11 @@ e. Headers through the coordinator's relay, with prefetch (part 1).
f. **Nothing enters the catalog without a usable name** (amended f. **Nothing enters the catalog without a usable name** (amended
2026-10-26, the user's direction; before: random-named releases keep 2026-10-26, the user's direction; before: random-named releases keep
no chunk but their rows). A release the window closed on whose files no chunk but their rows). A release the window closed on whose files
all have random names is no release yet: it goes to `pending_releases` all have random names (`isUnreadableName`, `names::is_unreadable_name`:
a name read token by token, so separators, spaces and random
"extensions" don't hide a scramble; since 2026-11-02, LOG.md (129),
when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu"
and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases`
(migration 6300, the HDD) with its keys, counts, article span, upload (migration 6300, the HDD) with its keys, counts, article span, upload
tool and probe targets, no row in `releases`, `release_texts` or tool and probe targets, no row in `releases`, `release_texts` or
`posters`, no chunk. The pending ladder works them **from least to most `posters`, no chunk. The pending ladder works them **from least to most
+83
View File
@@ -0,0 +1,83 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { isUnreadableName } from '#lib/release/index'
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
const CURSOR = 'readable'
/**
* The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
* readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
* false, so the readable filter hides them (an inner name from the prober still shows them). By the release name,
* which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
* Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
*/
export default class ReleasesReadable extends BaseCommand {
static commandName = 'releases:readable'
static description =
'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
static options: CommandOptions = { startApp: true }
@flags.number({ description: 'Release ids per batch' })
declare batch: number
@flags.number({ description: 'Stop after this id (a sample)' })
declare toId: number
@flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
declare apply: boolean
@flags.boolean({ description: 'Start from the first id, not the cursor' })
declare restart: boolean
async run() {
const batch = this.batch ?? 20_000
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
CURSOR,
])
let after =
this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
let looked = 0
let changed = 0
const sample: string[] = []
const started = Date.now()
while (after < hi) {
const to = Math.min(after + batch, hi)
const { rows } = await db.rawQuery(
`SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
[after, to]
)
looked += rows.length
const ids: number[] = []
for (const r of rows) {
if (!isUnreadableName(r.name)) continue
ids.push(Number(r.id))
if (sample.length < 40) sample.push(r.name)
}
changed += ids.length
if (this.apply) {
if (ids.length > 0)
await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
ids,
])
await db.rawQuery(
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
[CURSOR, String(to)]
)
}
after = to
if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
this.logger.info(
`Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
)
}
}
this.logger.info(
`${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
)
if (!this.apply) for (const n of sample) this.logger.info(` ${n}`)
}
}
+2 -2
View File
@@ -5,7 +5,7 @@
use crate::codec::{encode_chunk, ChunkFile}; use crate::codec::{encode_chunk, ChunkFile};
use crate::js::{self, sha1_hex}; use crate::js::{self, sha1_hex};
use crate::names::{self, is_random_file, looks_scrambled, JudgedFile, Release, Rules, Verdict}; use crate::names::{self, is_random_file, is_unreadable_name, looks_scrambled, JudgedFile, Release, Rules, Verdict};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::rc::Rc; use std::rc::Rc;
@@ -1560,7 +1560,7 @@ fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
bytes, bytes,
first_article, first_article,
last_article, last_article,
readable: chunk_files.iter().any(|f| !is_random_file(f.name)), readable: chunk_files.iter().any(|f| !is_unreadable_name(f.name)),
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)), has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
finished: true, finished: true,
edge: false, edge: false,
+188
View File
@@ -1065,6 +1065,169 @@ pub fn is_random_file(name: &str) -> bool {
looks_random(&file_stem(name)) looks_random(&file_stem(name))
} }
/// Readable or not (lib/release/readable.ts): whether a name says what a release is, read token by token, so
/// separators, spaces and random "extensions" don't hide a scramble that looks_random's single-token shapes miss
/// (LOG.md (129)). The readable flag; the drop rules keep looks_random, which is narrower.
/// Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed words.
static NOISE_TOKEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)^(addon|samples?|proof|subs?|nfo|part[0-9]+|vol[0-9]+|description|repost|rep|extras?|bonus|cd[0-9]*|dvd[0-9]*|dis[ck][0-9]*|par2|rar|zip|7z|mkv|mp4|avi|vol)$"));
/// Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th.
static CODE_TOKEN: LazyLock<Regex> = LazyLock::new(|| re(r"^([sS][0-9]{1,2}[eE][0-9]{1,3}|[xh]26[45]|[A-Za-z]+[0-9]+([a-z]{1,3}|x(64|86))?|[0-9]+[A-Za-z]{1,3}|[0-9]+[A-Za-z]+[0-9]+)$"));
/// The NZBSPLIT tools' hash between their markers is no part of the name.
static NZBSPLIT_HASH: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)__NZBS[P]LIT__[0-9a-f]{32}__NZBS[P]LIT__")); // "SP" is re()'s space class
/// An inner run of 2+ capitals with lowercase on both sides ("tBWTuz", "YDoaROv"; not "3DMark03").
static INNER_UPPER: LazyLock<Regex> = LazyLock::new(|| re(r"[a-z][A-Z]{2,}[a-z]"));
/// Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there.
fn scene_case(t: &[u8]) -> Vec<u8> {
let mut out = t.to_vec();
for i in 1..t.len().saturating_sub(1) {
if t[i] == b'i' && t[i - 1].is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() {
out[i] = b'I';
}
}
out
}
/// The signals of a scrambled token (readable.ts signals()).
struct TokenSignals {
/// Letter/digit and lower-to-upper switches.
switches: usize,
/// Case switches in both directions.
case_switches: usize,
/// Lone lowercase letters (not "i") between capitals ("NtpUgCmS").
lone: usize,
inner_upper: bool,
max_consonants: usize,
/// Vowels, a lowercase y too ("McCartney").
vowels: usize,
/// Runs of one character class.
runs: usize,
letters: usize,
/// The longest lowercase run.
max_lower: usize,
}
fn token_signals(t: &[u8]) -> TokenSignals {
let is_vowel = |c: u8| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u') || c == b'y';
let class = |c: u8| if c.is_ascii_digit() { 0 } else if c.is_ascii_lowercase() { 1 } else { 2 };
let mut s = TokenSignals { switches: 0, case_switches: 0, lone: 0, inner_upper: false, max_consonants: 0, vowels: 0, runs: 0, letters: 0, max_lower: 0 };
let mut consonants = 0;
let mut lower = 0;
for i in 0..t.len() {
let c = t[i];
if c.is_ascii_alphabetic() && !is_vowel(c) {
consonants += 1;
s.max_consonants = s.max_consonants.max(consonants);
} else {
consonants = 0;
}
if is_vowel(c) {
s.vowels += 1;
}
if c.is_ascii_alphabetic() {
s.letters += 1;
}
if c.is_ascii_lowercase() {
lower += 1;
s.max_lower = s.max_lower.max(lower);
} else {
lower = 0;
}
if i == 0 || class(t[i - 1]) != class(c) {
s.runs += 1;
}
if i == 0 {
continue;
}
let a = t[i - 1];
if a.is_ascii_digit() != c.is_ascii_digit() {
s.switches += 1;
} else if a.is_ascii_lowercase() && c.is_ascii_uppercase() {
s.switches += 1;
}
if (a.is_ascii_lowercase() && c.is_ascii_uppercase()) || (a.is_ascii_uppercase() && c.is_ascii_lowercase()) {
s.case_switches += 1;
}
if i < t.len() - 1 && c.is_ascii_lowercase() && c != b'i' && a.is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() {
s.lone += 1;
}
}
s.inner_upper = INNER_UPPER.is_match(t);
s
}
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
pub enum TokenKind {
Word,
Random,
Neutral,
}
/// What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a
/// packaging word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with readable.ts.
pub fn token_kind(t: &[u8]) -> TokenKind {
let n = t.len();
if t.iter().all(u8::is_ascii_digit) || NOISE_TOKEN.is_match(t) || CODE_TOKEN.is_match(t) {
return TokenKind::Neutral;
}
if t.iter().all(u8::is_ascii_alphabetic) {
let s = token_signals(&scene_case(t));
let capitalized = t[0].is_ascii_uppercase() && t[1..].iter().all(u8::is_ascii_lowercase);
if s.case_switches == 0 || (s.case_switches == 1 && capitalized) {
// A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word.
if n >= 10 && s.vowels * 6 < n {
return TokenKind::Random;
}
return if n >= 3 { TokenKind::Word } else { TokenKind::Neutral };
}
// Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs.
if n >= 6 && s.case_switches >= 2 && (s.inner_upper || (s.vowels * 4 < n && (s.runs >= 4 || s.lone >= 1))) {
return TokenKind::Random;
}
return TokenKind::Neutral;
}
if n >= 8 && t.iter().all(u8::is_ascii_hexdigit) && t.iter().filter(|c| c.is_ascii_digit()).count() >= 2 {
return TokenKind::Random;
}
if t.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) {
// 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay.
return if n >= 12 && token_signals(t).switches >= 6 { TokenKind::Random } else { TokenKind::Neutral };
}
let s = token_signals(&scene_case(t));
if n >= 5 && s.switches >= 3 {
return TokenKind::Random;
}
if n >= 6 && s.switches >= 2 && (s.lone >= 1 || s.inner_upper || s.max_consonants >= 4) {
return TokenKind::Random;
}
// Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus").
if n >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.max_lower <= 2 {
return TokenKind::Random;
}
TokenKind::Neutral
}
/// A file or release name no one can read (readable.ts isUnreadableName): looks_random's shapes, or more scrambled
/// tokens than words by letter count ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not
/// "Paul McCartney - Band on the Run", "ShesSoWild - Ester Ana 1080p").
pub fn is_unreadable_name(name: &str) -> bool {
let stem = file_stem(name);
let stem = NZBSPLIT_HASH.replace(stem.as_bytes(), &b" "[..]);
if looks_random(&String::from_utf8_lossy(&stem)) {
return true;
}
let mut words = 0;
let mut random = 0;
for t in stem.split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) {
match token_kind(t) {
TokenKind::Word => words += t.len(),
TokenKind::Random => random += t.len(),
TokenKind::Neutral => {}
}
}
random > 0 && random > words
}
/// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case /// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case
/// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"). /// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa").
/// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators. /// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators.
@@ -1622,6 +1785,31 @@ mod tests {
assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key); assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key);
} }
#[test]
fn unreadable_names() {
use super::is_unreadable_name;
// Scrambles with separators, spaces, a random "extension", a packaging word, a hex with an addon (LOG.md (129)).
for n in [
"Zx5AjNaK-mh7CFg6Hf8uu.part01.rar", "yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU", "PsNdnYiugfZ.rar", "phfplC9y07U2yrj", "n5d1ueIf34X",
"tJqr1tq7xK", "Description - Lt5nHjlo3RiuBb3C.par2", "bd5b4a59f3aa87114dcf75c0-addon.rar", "mrrtmzrzmi4dgmrzmyytqmlbztczbx",
"AA-NaR16302830434zxP32108309f3Ab", "_QBs01_MqZYnYwM_iAyRquhc-", "sgkra vU7u2yuw9LDLCjDu04pOez", "Jk5Pud.zQdNj", "7uKsD_NRgT-G_Uc_p8",
"NtpUgCmS_Xa1B2Pk", "{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}", "IkYovjrtZrItjQ.mkv", "ff75ce3f16c746c18b2d78cf.6c52f",
] {
assert!(is_unreadable_name(n), "{n}");
}
// Real names: scene tags, CamelCase, codes, leet, an acronym, a hash beside words.
for n in [
"Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune", "Paul McCartney - Band on the Run",
"ShesSoWild - Ester Ana 1080p", "MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS", "cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar",
"Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing", "Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON",
"Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr", "Eclipse_IDE_2023-12_R_win32_x86_64.zip", "PartsHPGL 1.06c", "Go4Customer Fax Machine v1.2b",
"AdBeGone 1.2", "3DMark03 320", "COD4MW2REMASTERED", "Movie.2020.1080p-GRP [a3f9e2b1]", "EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps",
"iPhone", "Some.Show.S01E01.1080p.mkv",
] {
assert!(!is_unreadable_name(n), "{n}");
}
}
#[test] #[test]
fn random_names() { fn random_names() {
assert!(looks_random("prvjr7hapd0i1ny14pz4")); assert!(looks_random("prvjr7hapd0i1ny14pz4"));
+2 -1
View File
@@ -2,6 +2,7 @@ import { hash } from 'node:crypto'
import type { Overview } from '../nntp/index.js' import type { Overview } from '../nntp/index.js'
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js' import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js' import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
import { isUnreadableName } from '../release/index.js'
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js' import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
import { encodeChunk, type ChunkFile } from '../chunks/index.js' import { encodeChunk, type ChunkFile } from '../chunks/index.js'
@@ -458,7 +459,7 @@ export function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[], uploa
bytes, bytes,
firstArticle, firstArticle,
lastArticle, lastArticle,
readable: chunkFiles.some((f) => !isRandomFile(f.name)), readable: chunkFiles.some((f) => !isUnreadableName(f.name)),
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)), hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
partial: files.some((f) => f.segments.length < f.segments[0].totalParts), partial: files.some((f) => f.segments.length < f.segments[0].totalParts),
payload: encodeChunk(chunkFiles), payload: encodeChunk(chunkFiles),
+2 -2
View File
@@ -5,7 +5,7 @@ import { parseRarHeaders } from '#lib/rar/index'
import { parseSevenZipTail } from '#lib/sevenzip/index' import { parseSevenZipTail } from '#lib/sevenzip/index'
import { parseZipStart } from '#lib/zip/index' import { parseZipStart } from '#lib/zip/index'
import { sniffContentType, type ContentType } from '#lib/content/index' import { sniffContentType, type ContentType } from '#lib/content/index'
import { fileStem, looksRandom } from '#lib/release/index' import { isUnreadableName } from '#lib/release/index'
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */ /** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
export interface Inspection { export interface Inspection {
@@ -139,7 +139,7 @@ export function usableName(inspection: Inspection, kind: string): string | null
const usable = (n: string | undefined) => { const usable = (n: string | undefined) => {
if (!n) return null if (!n) return null
const name = n.replace(/^.*[\\/]/, '').slice(0, 255) const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null return name && !PACKAGING.test(name) && !isUnreadableName(name) ? name : null
} }
const named = (inspection.names ?? []).map(usable).find((n) => n) const named = (inspection.names ?? []).map(usable).find((n) => n)
if (named) return named if (named) return named
+1
View File
@@ -3,3 +3,4 @@
*/ */
export { releaseIdentity, displayName, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js' export { releaseIdentity, displayName, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js'
export { looksRandom } from './random.js' export { looksRandom } from './random.js'
export { isUnreadableName, tokenKind } from './readable.js'
+136
View File
@@ -0,0 +1,136 @@
import { fileStem } from './identity.js'
import { looksRandom } from './random.js'
/**
* Readable or not: whether a name says what a release is (the catalog's readable flag, the prober's inner names).
* looksRandom (random.ts) knows the single-token shapes; this reads a name token by token, so separators, spaces
* and random "extensions" don't hide a scramble (LOG.md (129)).
*/
/**
* Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed
* words ("Description - Lt5nHjlo3RiuBb3C", "bd5b4a59f3aa87114dcf75c0-addon").
*/
const NOISE =
/^(addon|samples?|proof|subs?|nfo|part\d+|vol\d+|description|repost|rep|extras?|bonus|cd\d*|dvd\d*|dis[ck]\d*|par2|rar|zip|7z|mkv|mp4|avi|vol)$/i
/** Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th. */
const CODE =
/^([sS]\d{1,2}[eE]\d{1,3}|[xh]26[45]|[A-Za-z]+\d+([a-z]{1,3}|x(64|86))?|\d+[A-Za-z]{1,3}|\d+[A-Za-z]+\d+)$/
/** The NZBSPLIT tools' hash between their markers is no part of the name. */
const NZBSPLIT = /__NZBSPLIT__[0-9a-f]{32}__NZBSPLIT__/i
/** Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there. */
function sceneCase(t: string): string {
let out = ''
for (let i = 0; i < t.length; i++) {
out +=
t[i] === 'i' && i > 0 && i < t.length - 1 && /[A-Z]/.test(t[i - 1]) && /[A-Z]/.test(t[i + 1])
? 'I'
: t[i]
}
return out
}
/**
* The signals of a scrambled token: letter/digit and lower-to-upper switches; case switches in both directions; lone
* lowercase letters between capitals ("NtpUgCmS"); an inner run of 2+ capitals with lowercase on both sides
* ("tBWTuz", "YDoaROv"); the longest consonant run; vowels (a lowercase y too: "McCartney"); runs of one character class.
*/
function signals(t: string) {
let switches = 0
let caseSwitches = 0
let lone = 0
let consonants = 0
let maxConsonants = 0
for (let i = 0; i < t.length; i++) {
const c = t[i]
if (/[b-df-hj-np-tv-z]/i.test(c)) maxConsonants = Math.max(maxConsonants, ++consonants)
else consonants = 0
if (i === 0) continue
const a = t[i - 1]
if (/\d/.test(a) !== /\d/.test(c)) switches++
else if (/[a-z]/.test(a) && /[A-Z]/.test(c)) switches++
if ((/[a-z]/.test(a) && /[A-Z]/.test(c)) || (/[A-Z]/.test(a) && /[a-z]/.test(c))) caseSwitches++
if (
i < t.length - 1 &&
/[a-z]/.test(c) &&
c !== 'i' &&
/[A-Z]/.test(a) &&
/[A-Z]/.test(t[i + 1])
)
lone++
}
const innerUpper = /[a-z][A-Z]{2,}[a-z]/.test(t)
const vowels = (t.match(/[aeiouAEIOU]|y/g) ?? []).length
const runs = (t.match(/[a-z]+|[A-Z]+|\d+/g) ?? []).length
const letters = (t.match(/[A-Za-z]/g) ?? []).length
const maxLower = Math.max(0, ...(t.match(/[a-z]+/g) ?? []).map((r) => r.length))
return {
switches,
caseSwitches,
lone,
innerUpper,
maxConsonants,
vowels,
runs,
letters,
maxLower,
}
}
/**
* What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a packaging
* word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with names.rs token_kind.
*/
export function tokenKind(t: string): 'word' | 'random' | null {
if (/^\d+$/.test(t) || NOISE.test(t) || CODE.test(t)) return null
if (/^[A-Za-z]+$/.test(t)) {
const s = signals(sceneCase(t))
if (s.caseSwitches === 0 || (s.caseSwitches === 1 && /^[A-Z][a-z]+$/.test(t))) {
// A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word.
if (t.length >= 10 && s.vowels * 6 < t.length) return 'random'
return t.length >= 3 ? 'word' : null
}
// Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs.
if (
t.length >= 6 &&
s.caseSwitches >= 2 &&
(s.innerUpper || (s.vowels * 4 < t.length && (s.runs >= 4 || s.lone >= 1)))
)
return 'random'
return null
}
if (t.length >= 8 && /^[0-9a-f]+$/i.test(t) && (t.match(/\d/g)?.length ?? 0) >= 2) return 'random'
if (/^[A-Z0-9]+$/.test(t)) {
// 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay.
return t.length >= 12 && signals(t).switches >= 6 ? 'random' : null
}
const s = signals(sceneCase(t))
if (t.length >= 5 && s.switches >= 3) return 'random'
if (t.length >= 6 && s.switches >= 2 && (s.lone >= 1 || s.innerUpper || s.maxConsonants >= 4))
return 'random'
// Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus").
if (t.length >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.maxLower <= 2)
return 'random'
return null
}
/**
* A file or release name no one can read: looksRandom's shapes, or more scrambled tokens than words by letter count
* ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not "Paul McCartney - Band on the Run",
* "ShesSoWild - Ester Ana 1080p"). Separators and a random "extension" of any length don't hide a scramble, which
* looksRandom's single-token shapes miss (15% of the readable catalog on 2026-11-02; LOG.md (129)). The readable
* flag; the drop rules keep looksRandom, which is narrower.
*/
export function isUnreadableName(name: string): boolean {
const stem = fileStem(name).replace(NZBSPLIT, ' ')
if (looksRandom(stem)) return true
let words = 0
let random = 0
for (const t of stem.split(/[^A-Za-z0-9]+/)) {
if (!t) continue
const kind = tokenKind(t)
if (kind === 'word') words += t.length
else if (kind === 'random') random += t.length
}
return random > 0 && random > words
}
+6 -6
View File
@@ -431,10 +431,10 @@ test.group('BatchStore', (group) => {
assert, assert,
}) => { }) => {
await storeIn(GROUP, 1, [ await storeIn(GROUP, 1, [
...upload(10, 'TsIgNBewuFbuwWy.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'), ...upload(10, 'Some.Show.S01E04.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'),
...upload(20, 'TsIgNBewuFbuwWy.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'), ...upload(20, 'Some.Show.S01E04.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'),
...upload(30, 'TsIgNBewuFbuwWy.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'), ...upload(30, 'Some.Show.S01E04.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'),
...upload(40, 'TsIgNBewuFbuwWy.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'), ...upload(40, 'Some.Show.S01E04.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'),
...upload( ...upload(
50, 50,
'Some.Show.S01E03.part01.rar', 'Some.Show.S01E03.part01.rar',
@@ -455,8 +455,8 @@ test.group('BatchStore', (group) => {
candidates.map((c) => [c.kind, c.file]), candidates.map((c) => [c.kind, c.file]),
[ [
['archive', 'Some.Show.S01E03.part01.rar'], ['archive', 'Some.Show.S01E03.part01.rar'],
['archive', 'TsIgNBewuFbuwWy.7z.001'], ['archive', 'Some.Show.S01E04.7z.001'],
['7z-tail', 'TsIgNBewuFbuwWy.7z.004'], ['7z-tail', 'Some.Show.S01E04.7z.004'],
] ]
) )
}) })
+72
View File
@@ -0,0 +1,72 @@
import { test } from '@japa/runner'
import { isUnreadableName, tokenKind } from '#lib/release/index'
test.group('release / isUnreadableName', () => {
test('scrambles with separators, spaces, random extensions, packaging words and hex with an addon are unreadable', ({
assert,
}) => {
for (const n of [
'Zx5AjNaK-mh7CFg6Hf8uu.part01.rar',
'yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU',
'PsNdnYiugfZ.rar',
'phfplC9y07U2yrj',
'n5d1ueIf34X',
'tJqr1tq7xK',
'Description - Lt5nHjlo3RiuBb3C.par2',
'bd5b4a59f3aa87114dcf75c0-addon.rar',
'mrrtmzrzmi4dgmrzmyytqmlbztczbx',
'AA-NaR16302830434zxP32108309f3Ab',
'_QBs01_MqZYnYwM_iAyRquhc-',
'sgkra vU7u2yuw9LDLCjDu04pOez',
'Jk5Pud.zQdNj',
'7uKsD_NRgT-G_Uc_p8',
'NtpUgCmS_Xa1B2Pk',
'{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}',
'IkYovjrtZrItjQ.mkv',
'ff75ce3f16c746c18b2d78cf.6c52f',
'prvjr7hapd0i1ny14pz4',
]) {
assert.isTrue(isUnreadableName(n), n)
}
})
test('scene tags, CamelCase, codes, leet, acronyms and a hash beside words are readable', ({
assert,
}) => {
for (const n of [
'Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune',
'Paul McCartney - Band on the Run',
'ShesSoWild - Ester Ana 1080p',
'MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS',
'cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar',
'Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing',
'Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON',
'Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr',
'Eclipse_IDE_2023-12_R_win32_x86_64.zip',
'PartsHPGL 1.06c',
'Go4Customer Fax Machine v1.2b',
'AdBeGone 1.2',
'3DMark03 320',
'COD4MW2REMASTERED',
'Movie.2020.1080p-GRP [a3f9e2b1]',
'EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps',
'iPhone',
'Some.Show.S01E01.1080p.mkv',
]) {
assert.isFalse(isUnreadableName(n), n)
}
})
test('token kinds', ({ assert }) => {
assert.equal(tokenKind('Champions'), 'word')
assert.equal(tokenKind('PRESCRiPTiON'), 'word')
assert.equal(tokenKind('GarciaLive'), null)
assert.equal(tokenKind('S02E10'), null)
assert.equal(tokenKind('1080p'), null)
assert.equal(tokenKind('addon'), null)
assert.equal(tokenKind('NtpUgCmS'), 'random')
assert.equal(tokenKind('tBWTuz'), 'random')
assert.equal(tokenKind('Lt5nHjlo3RiuBb3C'), 'random')
assert.equal(tokenKind('a3f9e2b1'), 'random')
})
})