From 8e0747865981f4d32515e0bba58f1753bd4ecd60 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 12 Oct 2026 10:12:28 +0200 Subject: [PATCH] Counters: a total padded wider than its index is none; report diff on shared ranges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - "Very Best Of The Who [01/010]", "[02/020]", "[07/070]": a posting tool's total of index×10, so every track had a total of its own and was a release of its own. A total zero-padded wider than its index is no counter ("[001/010]" still is). - corpus_sweep.py report compares builds over the ranges both replayed (the corpus grows between builds). Co-Authored-By: Claude Opus 5.5 --- deploy/corpus_sweep.py | 27 ++++++++++++++++++--------- edge/src/names.rs | 19 ++++++++++++++++++- lib/release/identity.ts | 12 ++++++++++-- tests/unit/release/identity.spec.ts | 7 +++++++ 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/deploy/corpus_sweep.py b/deploy/corpus_sweep.py index f9b9d86..ae391e8 100644 --- a/deploy/corpus_sweep.py +++ b/deploy/corpus_sweep.py @@ -113,13 +113,20 @@ def replay(): shutil.rmtree(os.path.join(OUT, b)) -def load(build): - """Releases of one build's replay, merged by (group, key) as the catalog does.""" +def replayed(build): + """The ranges a build replayed, as paths relative to its directory.""" + root = os.path.join(OUT, build) + return {os.path.relpath(os.path.join(d, f), root) for d, _, files in os.walk(root) for f in files if f.endswith('.jsonl')} + + +def load(build, only=None): + """Releases of one build's replay (of the ranges in `only`, when given), merged by (group, key) as the catalog does.""" releases = {} - for dirpath, _, files in os.walk(os.path.join(OUT, build)): - for f in files: - if not f.endswith('.jsonl'): - continue + for rel in sorted(replayed(build)): + if only is not None and rel not in only: + continue + dirpath, f = os.path.split(os.path.join(OUT, build, rel)) + if True: for line in open(os.path.join(dirpath, f), encoding='utf-8', errors='replace'): r = json.loads(line) k = (r['group'], r['key']) @@ -182,10 +189,12 @@ def report(): show(r, ' ') if len(builds) >= 2: - before = load(builds[-2][1]) - print(f'\n== change since build {builds[-2][1]}: {len(before):,} → {len(rel):,} releases') + # Over the ranges both replayed: the corpus grows between builds. + both = replayed(builds[-2][1]) & replayed(build) + before, after = load(builds[-2][1], both), load(build, both) + print(f'\n== change since build {builds[-2][1]} on the {len(both)} ranges both replayed: {len(before):,} → {len(after):,} releases') names = lambda rs: collections.Counter((r['group'], r['name']) for r in rs.values()) - a, b = names(before), names(rel) + a, b = names(before), names(after) moved = sorted(set(a) | set(b), key=lambda k: -abs(a[k] - b[k]))[:20] for k in moved: if a[k] != b[k]: diff --git a/edge/src/names.rs b/edge/src/names.rs index a803c74..054d5ff 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -385,8 +385,9 @@ fn finish_remainder(s: &str) -> String { pub fn counter_total(subject: &str) -> Option { let s = replace("ED, subject, " ", true); if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) { + let index = c.get(1).or(c.get(3)).unwrap().as_bytes(); let total = c.get(2).or(c.get(4)).unwrap().as_bytes(); - return Some(js_number_string(number(total))); + return (!broken_total(index, total)).then(|| js_number_string(number(total))); } // Dates are no counters ("08-29-2011" read as 8 of 29). let undated = replace(&DATES, &s, " ", true); @@ -959,6 +960,11 @@ fn name_like(b: &[u8]) -> bool { }) } +/// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal). +fn broken_total(index: &[u8], total: &[u8]) -> bool { + total.len() > 1 && total[0] == b'0' && total.len() > index.len() +} + /// The second number of the first \[(\d+)/(\d+)\]. fn file_counter_total(s: &str) -> Option { let b = s.as_bytes(); @@ -973,6 +979,9 @@ fn file_counter_total(s: &str) -> Option { } let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); if d > 0 && b.get(slash + 1 + d) == Some(&b']') { + if broken_total(&b[i + 1..slash], &b[slash + 1..slash + 1 + d]) { + return None; + } return js::parse_int(&s[slash + 1..slash + 1 + d]); } } @@ -1283,6 +1292,14 @@ mod image_set_tests { assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key); } + #[test] + fn totals_padded_wider_than_the_index_are_no_counters() { + let who = |i: u32, n: &str| release_identity("p", &format!(r#"Very Best Of The Who [{i:02}/{:03}] - "{n}" yEnc"#, i * 10), n); + assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, "The Who - 02 - Anyway.mp3").key); + assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").file_total, None); + assert_eq!(super::counter_total(r#"x [001/010] - "a.rar""#).as_deref(), Some("10")); + } + #[test] fn dates_lead_no_numbered_items_and_repeated_names_go() { let show = |n: &str| release_identity("#sterntuary ", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n); diff --git a/lib/release/identity.ts b/lib/release/identity.ts index 38cf62a..5883596 100644 --- a/lib/release/identity.ts +++ b/lib/release/identity.ts @@ -193,10 +193,18 @@ export function imageSet(stem: string): string { return /(? 1 && total.startsWith('0') && total.length > index.length +} + export function counterTotal(subject: string): number | undefined { const s = subject.replace(/"[^"]*"/g, ' ') const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s) - if (plain) return Number(plain[2] ?? plain[4]) + if (plain) return brokenTotal(plain[1] ?? plain[3], plain[2] ?? plain[4]) ? undefined : Number(plain[2] ?? plain[4]) // Dates are no counters ("Avira AntiVir Rescue System 08-29-2011.rar", "… 27-02-2011", "18-08-2012"): // read as "8 of 29", such a release showed a wrong file total and counted as incomplete. const undated = s.replace(DATES, ' ') @@ -263,7 +271,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { const poster = randomPoster(input.poster) ? '' : input.poster const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject) // The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's. - const postTotal = counter ? Number.parseInt(counter[2], 10) : undefined + const postTotal = counter && !brokenTotal(counter[1], counter[2]) ? Number.parseInt(counter[2], 10) : undefined const stem = fileStem(input.name) if (input.weak) { diff --git a/tests/unit/release/identity.spec.ts b/tests/unit/release/identity.spec.ts index 944bbaa..3bd96f9 100644 --- a/tests/unit/release/identity.spec.ts +++ b/tests/unit/release/identity.spec.ts @@ -276,6 +276,13 @@ test.group('release / par2 names', () => { assert.notEqual(vlad('185', 77, 'P8250001.JPG').key, vlad('234', 77, 'P5300085.JPG').key) }) + test('a total padded wider than its index is no counter', ({ assert }) => { + const who = (i: number, n: string) => + releaseIdentity({ poster: 'p', subject: `Very Best Of The Who [${String(i).padStart(2, '0')}/${String(i * 10).padStart(3, '0')}] - "${n}" yEnc`, name: n, weak: false })! + assert.equal(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, 'The Who - 02 - Anyway.mp3').key) + assert.isUndefined(who(3, "The Who - 01 - I Can't Explain.mp3").fileTotal) + }) + test('a leading date numbers no item; a file name repeated in the subject goes everywhere', ({ assert }) => { const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })! const show = (n: string) => id(`#sterntuary - Jeff Rense Radio Program - "${n}" yEnc`, n, '#sterntuary ')