Readable names: a token-by-token classifier for the readable flag
Searches with the readable filter on returned scrambles: with a random
4-letter query nearly every hit was one ("Zx5AjNaK-mh7CFg6Hf8uu",
"Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ", "bd5b...-addon"), and
15% of a 6,000-row sample of the catalog's readable names were. looksRandom
knows single-token shapes only; separators, spaces, a random "extension"
of any length and a packaging word hid the rest.
isUnreadableName (lib/release/readable.ts) and names::is_unreadable_name
read a name token by token: words (one case or Capitalized), scrambles
(letter/digit and case switches, lone letters between capitals, inner
capital runs, few vowels, hex) and neutral tokens (digits, codes like
S02E10 and 1080p, CamelCase, packaging words); a name is unreadable when
its scrambles outweigh its words. Scene tags' "i" for "I" and the
NZBSPLIT hash are read as such. On the evaluation sets: 554 of 559
random-search names caught, 0 of 630 real names flagged (LOG.md (129)).
The edge's readable flag and the prober's inner names use it; the drop
rules keep looksRandom, which is narrower. An unnamed release goes to the
pending store (decision f), so more of the obfuscated groups' releases
wait for a probe now. releases:readable clears the flag on the existing
catalog's rows by name (a dry run by default).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
2643266559
commit
81a5dce45a
10 files changed
+497
-12
No files matched your search
+5
-1
@@ -879,7 +879,11 @@ e. Headers through the coordinator's relay, with prefetch (part 1).
|
|||||||
f. **Nothing enters the catalog without a usable name** (amended
|
f. **Nothing enters the catalog without a usable name** (amended
|
||||||
2026-10-26, the user's direction; before: random-named releases keep
|
2026-10-26, the user's direction; before: random-named releases keep
|
||||||
no chunk but their rows). A release the window closed on whose files
|
no chunk but their rows). A release the window closed on whose files
|
||||||
all have random names is no release yet: it goes to `pending_releases`
|
all have random names (`isUnreadableName`, `names::is_unreadable_name`:
|
||||||
|
a name read token by token, so separators, spaces and random
|
||||||
|
"extensions" don't hide a scramble; since 2026-11-02, LOG.md (129),
|
||||||
|
when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu"
|
||||||
|
and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases`
|
||||||
(migration 6300, the HDD) with its keys, counts, article span, upload
|
(migration 6300, the HDD) with its keys, counts, article span, upload
|
||||||
tool and probe targets, no row in `releases`, `release_texts` or
|
tool and probe targets, no row in `releases`, `release_texts` or
|
||||||
`posters`, no chunk. The pending ladder works them **from least to most
|
`posters`, no chunk. The pending ladder works them **from least to most
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||||
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import { isUnreadableName } from '#lib/release/index'
|
||||||
|
|
||||||
|
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
||||||
|
const CURSOR = 'readable'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
|
||||||
|
* readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
|
||||||
|
* false, so the readable filter hides them (an inner name from the prober still shows them). By the release name,
|
||||||
|
* which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
|
||||||
|
* Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
|
||||||
|
*/
|
||||||
|
export default class ReleasesReadable extends BaseCommand {
|
||||||
|
static commandName = 'releases:readable'
|
||||||
|
static description =
|
||||||
|
'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
|
||||||
|
static options: CommandOptions = { startApp: true }
|
||||||
|
|
||||||
|
@flags.number({ description: 'Release ids per batch' })
|
||||||
|
declare batch: number
|
||||||
|
|
||||||
|
@flags.number({ description: 'Stop after this id (a sample)' })
|
||||||
|
declare toId: number
|
||||||
|
|
||||||
|
@flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
|
||||||
|
declare apply: boolean
|
||||||
|
|
||||||
|
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
||||||
|
declare restart: boolean
|
||||||
|
|
||||||
|
async run() {
|
||||||
|
const batch = this.batch ?? 20_000
|
||||||
|
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
||||||
|
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
||||||
|
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
|
||||||
|
CURSOR,
|
||||||
|
])
|
||||||
|
let after =
|
||||||
|
this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
||||||
|
let looked = 0
|
||||||
|
let changed = 0
|
||||||
|
const sample: string[] = []
|
||||||
|
const started = Date.now()
|
||||||
|
while (after < hi) {
|
||||||
|
const to = Math.min(after + batch, hi)
|
||||||
|
const { rows } = await db.rawQuery(
|
||||||
|
`SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
|
||||||
|
[after, to]
|
||||||
|
)
|
||||||
|
looked += rows.length
|
||||||
|
const ids: number[] = []
|
||||||
|
for (const r of rows) {
|
||||||
|
if (!isUnreadableName(r.name)) continue
|
||||||
|
ids.push(Number(r.id))
|
||||||
|
if (sample.length < 40) sample.push(r.name)
|
||||||
|
}
|
||||||
|
changed += ids.length
|
||||||
|
if (this.apply) {
|
||||||
|
if (ids.length > 0)
|
||||||
|
await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
|
||||||
|
ids,
|
||||||
|
])
|
||||||
|
await db.rawQuery(
|
||||||
|
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||||
|
[CURSOR, String(to)]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
after = to
|
||||||
|
if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
|
||||||
|
this.logger.info(
|
||||||
|
`Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.logger.info(
|
||||||
|
`${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
|
||||||
|
)
|
||||||
|
if (!this.apply) for (const n of sample) this.logger.info(` ${n}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
+2
-2
@@ -5,7 +5,7 @@
|
|||||||
|
|
||||||
use crate::codec::{encode_chunk, ChunkFile};
|
use crate::codec::{encode_chunk, ChunkFile};
|
||||||
use crate::js::{self, sha1_hex};
|
use crate::js::{self, sha1_hex};
|
||||||
use crate::names::{self, is_random_file, looks_scrambled, JudgedFile, Release, Rules, Verdict};
|
use crate::names::{self, is_random_file, is_unreadable_name, looks_scrambled, JudgedFile, Release, Rules, Verdict};
|
||||||
use rustc_hash::{FxHashMap, FxHashSet};
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
@@ -1560,7 +1560,7 @@ fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
|
|||||||
bytes,
|
bytes,
|
||||||
first_article,
|
first_article,
|
||||||
last_article,
|
last_article,
|
||||||
readable: chunk_files.iter().any(|f| !is_random_file(f.name)),
|
readable: chunk_files.iter().any(|f| !is_unreadable_name(f.name)),
|
||||||
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
||||||
finished: true,
|
finished: true,
|
||||||
edge: false,
|
edge: false,
|
||||||
|
|||||||
@@ -1065,6 +1065,169 @@ pub fn is_random_file(name: &str) -> bool {
|
|||||||
looks_random(&file_stem(name))
|
looks_random(&file_stem(name))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Readable or not (lib/release/readable.ts): whether a name says what a release is, read token by token, so
|
||||||
|
/// separators, spaces and random "extensions" don't hide a scramble that looks_random's single-token shapes miss
|
||||||
|
/// (LOG.md (129)). The readable flag; the drop rules keep looks_random, which is narrower.
|
||||||
|
/// Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed words.
|
||||||
|
static NOISE_TOKEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)^(addon|samples?|proof|subs?|nfo|part[0-9]+|vol[0-9]+|description|repost|rep|extras?|bonus|cd[0-9]*|dvd[0-9]*|dis[ck][0-9]*|par2|rar|zip|7z|mkv|mp4|avi|vol)$"));
|
||||||
|
/// Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th.
|
||||||
|
static CODE_TOKEN: LazyLock<Regex> = LazyLock::new(|| re(r"^([sS][0-9]{1,2}[eE][0-9]{1,3}|[xh]26[45]|[A-Za-z]+[0-9]+([a-z]{1,3}|x(64|86))?|[0-9]+[A-Za-z]{1,3}|[0-9]+[A-Za-z]+[0-9]+)$"));
|
||||||
|
/// The NZBSPLIT tools' hash between their markers is no part of the name.
|
||||||
|
static NZBSPLIT_HASH: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)__NZBS[P]LIT__[0-9a-f]{32}__NZBS[P]LIT__")); // "SP" is re()'s space class
|
||||||
|
/// An inner run of 2+ capitals with lowercase on both sides ("tBWTuz", "YDoaROv"; not "3DMark03").
|
||||||
|
static INNER_UPPER: LazyLock<Regex> = LazyLock::new(|| re(r"[a-z][A-Z]{2,}[a-z]"));
|
||||||
|
|
||||||
|
/// Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there.
|
||||||
|
fn scene_case(t: &[u8]) -> Vec<u8> {
|
||||||
|
let mut out = t.to_vec();
|
||||||
|
for i in 1..t.len().saturating_sub(1) {
|
||||||
|
if t[i] == b'i' && t[i - 1].is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() {
|
||||||
|
out[i] = b'I';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The signals of a scrambled token (readable.ts signals()).
|
||||||
|
struct TokenSignals {
|
||||||
|
/// Letter/digit and lower-to-upper switches.
|
||||||
|
switches: usize,
|
||||||
|
/// Case switches in both directions.
|
||||||
|
case_switches: usize,
|
||||||
|
/// Lone lowercase letters (not "i") between capitals ("NtpUgCmS").
|
||||||
|
lone: usize,
|
||||||
|
inner_upper: bool,
|
||||||
|
max_consonants: usize,
|
||||||
|
/// Vowels, a lowercase y too ("McCartney").
|
||||||
|
vowels: usize,
|
||||||
|
/// Runs of one character class.
|
||||||
|
runs: usize,
|
||||||
|
letters: usize,
|
||||||
|
/// The longest lowercase run.
|
||||||
|
max_lower: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn token_signals(t: &[u8]) -> TokenSignals {
|
||||||
|
let is_vowel = |c: u8| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u') || c == b'y';
|
||||||
|
let class = |c: u8| if c.is_ascii_digit() { 0 } else if c.is_ascii_lowercase() { 1 } else { 2 };
|
||||||
|
let mut s = TokenSignals { switches: 0, case_switches: 0, lone: 0, inner_upper: false, max_consonants: 0, vowels: 0, runs: 0, letters: 0, max_lower: 0 };
|
||||||
|
let mut consonants = 0;
|
||||||
|
let mut lower = 0;
|
||||||
|
for i in 0..t.len() {
|
||||||
|
let c = t[i];
|
||||||
|
if c.is_ascii_alphabetic() && !is_vowel(c) {
|
||||||
|
consonants += 1;
|
||||||
|
s.max_consonants = s.max_consonants.max(consonants);
|
||||||
|
} else {
|
||||||
|
consonants = 0;
|
||||||
|
}
|
||||||
|
if is_vowel(c) {
|
||||||
|
s.vowels += 1;
|
||||||
|
}
|
||||||
|
if c.is_ascii_alphabetic() {
|
||||||
|
s.letters += 1;
|
||||||
|
}
|
||||||
|
if c.is_ascii_lowercase() {
|
||||||
|
lower += 1;
|
||||||
|
s.max_lower = s.max_lower.max(lower);
|
||||||
|
} else {
|
||||||
|
lower = 0;
|
||||||
|
}
|
||||||
|
if i == 0 || class(t[i - 1]) != class(c) {
|
||||||
|
s.runs += 1;
|
||||||
|
}
|
||||||
|
if i == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let a = t[i - 1];
|
||||||
|
if a.is_ascii_digit() != c.is_ascii_digit() {
|
||||||
|
s.switches += 1;
|
||||||
|
} else if a.is_ascii_lowercase() && c.is_ascii_uppercase() {
|
||||||
|
s.switches += 1;
|
||||||
|
}
|
||||||
|
if (a.is_ascii_lowercase() && c.is_ascii_uppercase()) || (a.is_ascii_uppercase() && c.is_ascii_lowercase()) {
|
||||||
|
s.case_switches += 1;
|
||||||
|
}
|
||||||
|
if i < t.len() - 1 && c.is_ascii_lowercase() && c != b'i' && a.is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() {
|
||||||
|
s.lone += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.inner_upper = INNER_UPPER.is_match(t);
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
|
||||||
|
pub enum TokenKind {
|
||||||
|
Word,
|
||||||
|
Random,
|
||||||
|
Neutral,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a
|
||||||
|
/// packaging word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with readable.ts.
|
||||||
|
pub fn token_kind(t: &[u8]) -> TokenKind {
|
||||||
|
let n = t.len();
|
||||||
|
if t.iter().all(u8::is_ascii_digit) || NOISE_TOKEN.is_match(t) || CODE_TOKEN.is_match(t) {
|
||||||
|
return TokenKind::Neutral;
|
||||||
|
}
|
||||||
|
if t.iter().all(u8::is_ascii_alphabetic) {
|
||||||
|
let s = token_signals(&scene_case(t));
|
||||||
|
let capitalized = t[0].is_ascii_uppercase() && t[1..].iter().all(u8::is_ascii_lowercase);
|
||||||
|
if s.case_switches == 0 || (s.case_switches == 1 && capitalized) {
|
||||||
|
// A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word.
|
||||||
|
if n >= 10 && s.vowels * 6 < n {
|
||||||
|
return TokenKind::Random;
|
||||||
|
}
|
||||||
|
return if n >= 3 { TokenKind::Word } else { TokenKind::Neutral };
|
||||||
|
}
|
||||||
|
// Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs.
|
||||||
|
if n >= 6 && s.case_switches >= 2 && (s.inner_upper || (s.vowels * 4 < n && (s.runs >= 4 || s.lone >= 1))) {
|
||||||
|
return TokenKind::Random;
|
||||||
|
}
|
||||||
|
return TokenKind::Neutral;
|
||||||
|
}
|
||||||
|
if n >= 8 && t.iter().all(u8::is_ascii_hexdigit) && t.iter().filter(|c| c.is_ascii_digit()).count() >= 2 {
|
||||||
|
return TokenKind::Random;
|
||||||
|
}
|
||||||
|
if t.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) {
|
||||||
|
// 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay.
|
||||||
|
return if n >= 12 && token_signals(t).switches >= 6 { TokenKind::Random } else { TokenKind::Neutral };
|
||||||
|
}
|
||||||
|
let s = token_signals(&scene_case(t));
|
||||||
|
if n >= 5 && s.switches >= 3 {
|
||||||
|
return TokenKind::Random;
|
||||||
|
}
|
||||||
|
if n >= 6 && s.switches >= 2 && (s.lone >= 1 || s.inner_upper || s.max_consonants >= 4) {
|
||||||
|
return TokenKind::Random;
|
||||||
|
}
|
||||||
|
// Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus").
|
||||||
|
if n >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.max_lower <= 2 {
|
||||||
|
return TokenKind::Random;
|
||||||
|
}
|
||||||
|
TokenKind::Neutral
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A file or release name no one can read (readable.ts isUnreadableName): looks_random's shapes, or more scrambled
|
||||||
|
/// tokens than words by letter count ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not
|
||||||
|
/// "Paul McCartney - Band on the Run", "ShesSoWild - Ester Ana 1080p").
|
||||||
|
pub fn is_unreadable_name(name: &str) -> bool {
|
||||||
|
let stem = file_stem(name);
|
||||||
|
let stem = NZBSPLIT_HASH.replace(stem.as_bytes(), &b" "[..]);
|
||||||
|
if looks_random(&String::from_utf8_lossy(&stem)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let mut words = 0;
|
||||||
|
let mut random = 0;
|
||||||
|
for t in stem.split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) {
|
||||||
|
match token_kind(t) {
|
||||||
|
TokenKind::Word => words += t.len(),
|
||||||
|
TokenKind::Random => random += t.len(),
|
||||||
|
TokenKind::Neutral => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
random > 0 && random > words
|
||||||
|
}
|
||||||
|
|
||||||
/// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case
|
/// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case
|
||||||
/// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa").
|
/// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa").
|
||||||
/// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators.
|
/// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators.
|
||||||
@@ -1622,6 +1785,31 @@ mod tests {
|
|||||||
assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key);
|
assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unreadable_names() {
|
||||||
|
use super::is_unreadable_name;
|
||||||
|
// Scrambles with separators, spaces, a random "extension", a packaging word, a hex with an addon (LOG.md (129)).
|
||||||
|
for n in [
|
||||||
|
"Zx5AjNaK-mh7CFg6Hf8uu.part01.rar", "yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU", "PsNdnYiugfZ.rar", "phfplC9y07U2yrj", "n5d1ueIf34X",
|
||||||
|
"tJqr1tq7xK", "Description - Lt5nHjlo3RiuBb3C.par2", "bd5b4a59f3aa87114dcf75c0-addon.rar", "mrrtmzrzmi4dgmrzmyytqmlbztczbx",
|
||||||
|
"AA-NaR16302830434zxP32108309f3Ab", "_QBs01_MqZYnYwM_iAyRquhc-", "sgkra vU7u2yuw9LDLCjDu04pOez", "Jk5Pud.zQdNj", "7uKsD_NRgT-G_Uc_p8",
|
||||||
|
"NtpUgCmS_Xa1B2Pk", "{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}", "IkYovjrtZrItjQ.mkv", "ff75ce3f16c746c18b2d78cf.6c52f",
|
||||||
|
] {
|
||||||
|
assert!(is_unreadable_name(n), "{n}");
|
||||||
|
}
|
||||||
|
// Real names: scene tags, CamelCase, codes, leet, an acronym, a hash beside words.
|
||||||
|
for n in [
|
||||||
|
"Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune", "Paul McCartney - Band on the Run",
|
||||||
|
"ShesSoWild - Ester Ana 1080p", "MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS", "cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar",
|
||||||
|
"Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing", "Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON",
|
||||||
|
"Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr", "Eclipse_IDE_2023-12_R_win32_x86_64.zip", "PartsHPGL 1.06c", "Go4Customer Fax Machine v1.2b",
|
||||||
|
"AdBeGone 1.2", "3DMark03 320", "COD4MW2REMASTERED", "Movie.2020.1080p-GRP [a3f9e2b1]", "EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps",
|
||||||
|
"iPhone", "Some.Show.S01E01.1080p.mkv",
|
||||||
|
] {
|
||||||
|
assert!(!is_unreadable_name(n), "{n}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn random_names() {
|
fn random_names() {
|
||||||
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { hash } from 'node:crypto'
|
|||||||
import type { Overview } from '../nntp/index.js'
|
import type { Overview } from '../nntp/index.js'
|
||||||
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
|
||||||
import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
|
import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
|
||||||
|
import { isUnreadableName } from '../release/index.js'
|
||||||
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
|
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
|
||||||
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
|
||||||
|
|
||||||
@@ -458,7 +459,7 @@ export function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[], uploa
|
|||||||
bytes,
|
bytes,
|
||||||
firstArticle,
|
firstArticle,
|
||||||
lastArticle,
|
lastArticle,
|
||||||
readable: chunkFiles.some((f) => !isRandomFile(f.name)),
|
readable: chunkFiles.some((f) => !isUnreadableName(f.name)),
|
||||||
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
|
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
|
||||||
partial: files.some((f) => f.segments.length < f.segments[0].totalParts),
|
partial: files.some((f) => f.segments.length < f.segments[0].totalParts),
|
||||||
payload: encodeChunk(chunkFiles),
|
payload: encodeChunk(chunkFiles),
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { parseRarHeaders } from '#lib/rar/index'
|
|||||||
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
import { parseSevenZipTail } from '#lib/sevenzip/index'
|
||||||
import { parseZipStart } from '#lib/zip/index'
|
import { parseZipStart } from '#lib/zip/index'
|
||||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||||
import { fileStem, looksRandom } from '#lib/release/index'
|
import { isUnreadableName } from '#lib/release/index'
|
||||||
|
|
||||||
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
||||||
export interface Inspection {
|
export interface Inspection {
|
||||||
@@ -139,7 +139,7 @@ export function usableName(inspection: Inspection, kind: string): string | null
|
|||||||
const usable = (n: string | undefined) => {
|
const usable = (n: string | undefined) => {
|
||||||
if (!n) return null
|
if (!n) return null
|
||||||
const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
|
const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
|
||||||
return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null
|
return name && !PACKAGING.test(name) && !isUnreadableName(name) ? name : null
|
||||||
}
|
}
|
||||||
const named = (inspection.names ?? []).map(usable).find((n) => n)
|
const named = (inspection.names ?? []).map(usable).find((n) => n)
|
||||||
if (named) return named
|
if (named) return named
|
||||||
|
|||||||
@@ -3,3 +3,4 @@
|
|||||||
*/
|
*/
|
||||||
export { releaseIdentity, displayName, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js'
|
export { releaseIdentity, displayName, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, sizeAfterName, runOn, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './identity.js'
|
||||||
export { looksRandom } from './random.js'
|
export { looksRandom } from './random.js'
|
||||||
|
export { isUnreadableName, tokenKind } from './readable.js'
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { fileStem } from './identity.js'
|
||||||
|
import { looksRandom } from './random.js'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Readable or not: whether a name says what a release is (the catalog's readable flag, the prober's inner names).
|
||||||
|
* looksRandom (random.ts) knows the single-token shapes; this reads a name token by token, so separators, spaces
|
||||||
|
* and random "extensions" don't hide a scramble (LOG.md (129)).
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed
|
||||||
|
* words ("Description - Lt5nHjlo3RiuBb3C", "bd5b4a59f3aa87114dcf75c0-addon").
|
||||||
|
*/
|
||||||
|
const NOISE =
|
||||||
|
/^(addon|samples?|proof|subs?|nfo|part\d+|vol\d+|description|repost|rep|extras?|bonus|cd\d*|dvd\d*|dis[ck]\d*|par2|rar|zip|7z|mkv|mp4|avi|vol)$/i
|
||||||
|
/** Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th. */
|
||||||
|
const CODE =
|
||||||
|
/^([sS]\d{1,2}[eE]\d{1,3}|[xh]26[45]|[A-Za-z]+\d+([a-z]{1,3}|x(64|86))?|\d+[A-Za-z]{1,3}|\d+[A-Za-z]+\d+)$/
|
||||||
|
/** The NZBSPLIT tools' hash between their markers is no part of the name. */
|
||||||
|
const NZBSPLIT = /__NZBSPLIT__[0-9a-f]{32}__NZBSPLIT__/i
|
||||||
|
|
||||||
|
/** Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there. */
|
||||||
|
function sceneCase(t: string): string {
|
||||||
|
let out = ''
|
||||||
|
for (let i = 0; i < t.length; i++) {
|
||||||
|
out +=
|
||||||
|
t[i] === 'i' && i > 0 && i < t.length - 1 && /[A-Z]/.test(t[i - 1]) && /[A-Z]/.test(t[i + 1])
|
||||||
|
? 'I'
|
||||||
|
: t[i]
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The signals of a scrambled token: letter/digit and lower-to-upper switches; case switches in both directions; lone
|
||||||
|
* lowercase letters between capitals ("NtpUgCmS"); an inner run of 2+ capitals with lowercase on both sides
|
||||||
|
* ("tBWTuz", "YDoaROv"); the longest consonant run; vowels (a lowercase y too: "McCartney"); runs of one character class.
|
||||||
|
*/
|
||||||
|
function signals(t: string) {
|
||||||
|
let switches = 0
|
||||||
|
let caseSwitches = 0
|
||||||
|
let lone = 0
|
||||||
|
let consonants = 0
|
||||||
|
let maxConsonants = 0
|
||||||
|
for (let i = 0; i < t.length; i++) {
|
||||||
|
const c = t[i]
|
||||||
|
if (/[b-df-hj-np-tv-z]/i.test(c)) maxConsonants = Math.max(maxConsonants, ++consonants)
|
||||||
|
else consonants = 0
|
||||||
|
if (i === 0) continue
|
||||||
|
const a = t[i - 1]
|
||||||
|
if (/\d/.test(a) !== /\d/.test(c)) switches++
|
||||||
|
else if (/[a-z]/.test(a) && /[A-Z]/.test(c)) switches++
|
||||||
|
if ((/[a-z]/.test(a) && /[A-Z]/.test(c)) || (/[A-Z]/.test(a) && /[a-z]/.test(c))) caseSwitches++
|
||||||
|
if (
|
||||||
|
i < t.length - 1 &&
|
||||||
|
/[a-z]/.test(c) &&
|
||||||
|
c !== 'i' &&
|
||||||
|
/[A-Z]/.test(a) &&
|
||||||
|
/[A-Z]/.test(t[i + 1])
|
||||||
|
)
|
||||||
|
lone++
|
||||||
|
}
|
||||||
|
const innerUpper = /[a-z][A-Z]{2,}[a-z]/.test(t)
|
||||||
|
const vowels = (t.match(/[aeiouAEIOU]|y/g) ?? []).length
|
||||||
|
const runs = (t.match(/[a-z]+|[A-Z]+|\d+/g) ?? []).length
|
||||||
|
const letters = (t.match(/[A-Za-z]/g) ?? []).length
|
||||||
|
const maxLower = Math.max(0, ...(t.match(/[a-z]+/g) ?? []).map((r) => r.length))
|
||||||
|
return {
|
||||||
|
switches,
|
||||||
|
caseSwitches,
|
||||||
|
lone,
|
||||||
|
innerUpper,
|
||||||
|
maxConsonants,
|
||||||
|
vowels,
|
||||||
|
runs,
|
||||||
|
letters,
|
||||||
|
maxLower,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a packaging
|
||||||
|
* word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with names.rs token_kind.
|
||||||
|
*/
|
||||||
|
export function tokenKind(t: string): 'word' | 'random' | null {
|
||||||
|
if (/^\d+$/.test(t) || NOISE.test(t) || CODE.test(t)) return null
|
||||||
|
if (/^[A-Za-z]+$/.test(t)) {
|
||||||
|
const s = signals(sceneCase(t))
|
||||||
|
if (s.caseSwitches === 0 || (s.caseSwitches === 1 && /^[A-Z][a-z]+$/.test(t))) {
|
||||||
|
// A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word.
|
||||||
|
if (t.length >= 10 && s.vowels * 6 < t.length) return 'random'
|
||||||
|
return t.length >= 3 ? 'word' : null
|
||||||
|
}
|
||||||
|
// Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs.
|
||||||
|
if (
|
||||||
|
t.length >= 6 &&
|
||||||
|
s.caseSwitches >= 2 &&
|
||||||
|
(s.innerUpper || (s.vowels * 4 < t.length && (s.runs >= 4 || s.lone >= 1)))
|
||||||
|
)
|
||||||
|
return 'random'
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
if (t.length >= 8 && /^[0-9a-f]+$/i.test(t) && (t.match(/\d/g)?.length ?? 0) >= 2) return 'random'
|
||||||
|
if (/^[A-Z0-9]+$/.test(t)) {
|
||||||
|
// 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay.
|
||||||
|
return t.length >= 12 && signals(t).switches >= 6 ? 'random' : null
|
||||||
|
}
|
||||||
|
const s = signals(sceneCase(t))
|
||||||
|
if (t.length >= 5 && s.switches >= 3) return 'random'
|
||||||
|
if (t.length >= 6 && s.switches >= 2 && (s.lone >= 1 || s.innerUpper || s.maxConsonants >= 4))
|
||||||
|
return 'random'
|
||||||
|
// Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus").
|
||||||
|
if (t.length >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.maxLower <= 2)
|
||||||
|
return 'random'
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A file or release name no one can read: looksRandom's shapes, or more scrambled tokens than words by letter count
|
||||||
|
* ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not "Paul McCartney - Band on the Run",
|
||||||
|
* "ShesSoWild - Ester Ana 1080p"). Separators and a random "extension" of any length don't hide a scramble, which
|
||||||
|
* looksRandom's single-token shapes miss (15% of the readable catalog on 2026-11-02; LOG.md (129)). The readable
|
||||||
|
* flag; the drop rules keep looksRandom, which is narrower.
|
||||||
|
*/
|
||||||
|
export function isUnreadableName(name: string): boolean {
|
||||||
|
const stem = fileStem(name).replace(NZBSPLIT, ' ')
|
||||||
|
if (looksRandom(stem)) return true
|
||||||
|
let words = 0
|
||||||
|
let random = 0
|
||||||
|
for (const t of stem.split(/[^A-Za-z0-9]+/)) {
|
||||||
|
if (!t) continue
|
||||||
|
const kind = tokenKind(t)
|
||||||
|
if (kind === 'word') words += t.length
|
||||||
|
else if (kind === 'random') random += t.length
|
||||||
|
}
|
||||||
|
return random > 0 && random > words
|
||||||
|
}
|
||||||
@@ -431,10 +431,10 @@ test.group('BatchStore', (group) => {
|
|||||||
assert,
|
assert,
|
||||||
}) => {
|
}) => {
|
||||||
await storeIn(GROUP, 1, [
|
await storeIn(GROUP, 1, [
|
||||||
...upload(10, 'TsIgNBewuFbuwWy.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'),
|
...upload(10, 'Some.Show.S01E04.7z.001', 2, '[1/4]', '2026-10-23T08:24:00Z'),
|
||||||
...upload(20, 'TsIgNBewuFbuwWy.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'),
|
...upload(20, 'Some.Show.S01E04.7z.002', 2, '[2/4]', '2026-10-23T08:26:00Z'),
|
||||||
...upload(30, 'TsIgNBewuFbuwWy.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'),
|
...upload(30, 'Some.Show.S01E04.7z.003', 2, '[3/4]', '2026-10-23T08:28:00Z'),
|
||||||
...upload(40, 'TsIgNBewuFbuwWy.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'),
|
...upload(40, 'Some.Show.S01E04.7z.004', 2, '[4/4]', '2026-10-23T08:30:00Z'),
|
||||||
...upload(
|
...upload(
|
||||||
50,
|
50,
|
||||||
'Some.Show.S01E03.part01.rar',
|
'Some.Show.S01E03.part01.rar',
|
||||||
@@ -455,8 +455,8 @@ test.group('BatchStore', (group) => {
|
|||||||
candidates.map((c) => [c.kind, c.file]),
|
candidates.map((c) => [c.kind, c.file]),
|
||||||
[
|
[
|
||||||
['archive', 'Some.Show.S01E03.part01.rar'],
|
['archive', 'Some.Show.S01E03.part01.rar'],
|
||||||
['archive', 'TsIgNBewuFbuwWy.7z.001'],
|
['archive', 'Some.Show.S01E04.7z.001'],
|
||||||
['7z-tail', 'TsIgNBewuFbuwWy.7z.004'],
|
['7z-tail', 'Some.Show.S01E04.7z.004'],
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import { isUnreadableName, tokenKind } from '#lib/release/index'
|
||||||
|
|
||||||
|
test.group('release / isUnreadableName', () => {
|
||||||
|
test('scrambles with separators, spaces, random extensions, packaging words and hex with an addon are unreadable', ({
|
||||||
|
assert,
|
||||||
|
}) => {
|
||||||
|
for (const n of [
|
||||||
|
'Zx5AjNaK-mh7CFg6Hf8uu.part01.rar',
|
||||||
|
'yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU',
|
||||||
|
'PsNdnYiugfZ.rar',
|
||||||
|
'phfplC9y07U2yrj',
|
||||||
|
'n5d1ueIf34X',
|
||||||
|
'tJqr1tq7xK',
|
||||||
|
'Description - Lt5nHjlo3RiuBb3C.par2',
|
||||||
|
'bd5b4a59f3aa87114dcf75c0-addon.rar',
|
||||||
|
'mrrtmzrzmi4dgmrzmyytqmlbztczbx',
|
||||||
|
'AA-NaR16302830434zxP32108309f3Ab',
|
||||||
|
'_QBs01_MqZYnYwM_iAyRquhc-',
|
||||||
|
'sgkra vU7u2yuw9LDLCjDu04pOez',
|
||||||
|
'Jk5Pud.zQdNj',
|
||||||
|
'7uKsD_NRgT-G_Uc_p8',
|
||||||
|
'NtpUgCmS_Xa1B2Pk',
|
||||||
|
'{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}',
|
||||||
|
'IkYovjrtZrItjQ.mkv',
|
||||||
|
'ff75ce3f16c746c18b2d78cf.6c52f',
|
||||||
|
'prvjr7hapd0i1ny14pz4',
|
||||||
|
]) {
|
||||||
|
assert.isTrue(isUnreadableName(n), n)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test('scene tags, CamelCase, codes, leet, acronyms and a hash beside words are readable', ({
|
||||||
|
assert,
|
||||||
|
}) => {
|
||||||
|
for (const n of [
|
||||||
|
'Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune',
|
||||||
|
'Paul McCartney - Band on the Run',
|
||||||
|
'ShesSoWild - Ester Ana 1080p',
|
||||||
|
'MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS',
|
||||||
|
'cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar',
|
||||||
|
'Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing',
|
||||||
|
'Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON',
|
||||||
|
'Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr',
|
||||||
|
'Eclipse_IDE_2023-12_R_win32_x86_64.zip',
|
||||||
|
'PartsHPGL 1.06c',
|
||||||
|
'Go4Customer Fax Machine v1.2b',
|
||||||
|
'AdBeGone 1.2',
|
||||||
|
'3DMark03 320',
|
||||||
|
'COD4MW2REMASTERED',
|
||||||
|
'Movie.2020.1080p-GRP [a3f9e2b1]',
|
||||||
|
'EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps',
|
||||||
|
'iPhone',
|
||||||
|
'Some.Show.S01E01.1080p.mkv',
|
||||||
|
]) {
|
||||||
|
assert.isFalse(isUnreadableName(n), n)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test('token kinds', ({ assert }) => {
|
||||||
|
assert.equal(tokenKind('Champions'), 'word')
|
||||||
|
assert.equal(tokenKind('PRESCRiPTiON'), 'word')
|
||||||
|
assert.equal(tokenKind('GarciaLive'), null)
|
||||||
|
assert.equal(tokenKind('S02E10'), null)
|
||||||
|
assert.equal(tokenKind('1080p'), null)
|
||||||
|
assert.equal(tokenKind('addon'), null)
|
||||||
|
assert.equal(tokenKind('NtpUgCmS'), 'random')
|
||||||
|
assert.equal(tokenKind('tBWTuz'), 'random')
|
||||||
|
assert.equal(tokenKind('Lt5nHjlo3RiuBb3C'), 'random')
|
||||||
|
assert.equal(tokenKind('a3f9e2b1'), 'random')
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in new issue
Block a user