Counters: a total padded wider than its index is none; report diff on shared ranges
Test and deploy / test (push) Successful in 49s
Test and deploy / edge (push) Successful in 2m1s
Test and deploy / deploy (push) Successful in 1m25s

- "Very Best Of The Who [01/010]", "[02/020]", "[07/070]": a posting tool's
  total of index×10, so every track had a total of its own and was a
  release of its own. A total zero-padded wider than its index is no
  counter ("[001/010]" still is).
- corpus_sweep.py report compares builds over the ranges both replayed (the
  corpus grows between builds).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-12 10:12:28 +02:00
1 parent 1a5035d988
commit 8e07478659
4 files changed
+53 -12

No files matched your search

+18 -9
View File
@@ -113,13 +113,20 @@ def replay():
shutil.rmtree(os.path.join(OUT, b))
def load(build):
"""Releases of one build's replay, merged by (group, key) as the catalog does."""
def replayed(build):
"""The ranges a build replayed, as paths relative to its directory."""
root = os.path.join(OUT, build)
return {os.path.relpath(os.path.join(d, f), root) for d, _, files in os.walk(root) for f in files if f.endswith('.jsonl')}
def load(build, only=None):
"""Releases of one build's replay (of the ranges in `only`, when given), merged by (group, key) as the catalog does."""
releases = {}
for dirpath, _, files in os.walk(os.path.join(OUT, build)):
for f in files:
if not f.endswith('.jsonl'):
continue
for rel in sorted(replayed(build)):
if only is not None and rel not in only:
continue
dirpath, f = os.path.split(os.path.join(OUT, build, rel))
if True:
for line in open(os.path.join(dirpath, f), encoding='utf-8', errors='replace'):
r = json.loads(line)
k = (r['group'], r['key'])
@@ -182,10 +189,12 @@ def report():
show(r, ' ')
if len(builds) >= 2:
before = load(builds[-2][1])
print(f'\n== change since build {builds[-2][1]}: {len(before):,} → {len(rel):,} releases')
# Over the ranges both replayed: the corpus grows between builds.
both = replayed(builds[-2][1]) & replayed(build)
before, after = load(builds[-2][1], both), load(build, both)
print(f'\n== change since build {builds[-2][1]} on the {len(both)} ranges both replayed: {len(before):,} → {len(after):,} releases')
names = lambda rs: collections.Counter((r['group'], r['name']) for r in rs.values())
a, b = names(before), names(rel)
a, b = names(before), names(after)
moved = sorted(set(a) | set(b), key=lambda k: -abs(a[k] - b[k]))[:20]
for k in moved:
if a[k] != b[k]:
+18 -1
View File
@@ -385,8 +385,9 @@ fn finish_remainder(s: &str) -> String {
pub fn counter_total(subject: &str) -> Option<String> {
let s = replace(&QUOTED, subject, " ", true);
if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) {
let index = c.get(1).or(c.get(3)).unwrap().as_bytes();
let total = c.get(2).or(c.get(4)).unwrap().as_bytes();
return Some(js_number_string(number(total)));
return (!broken_total(index, total)).then(|| js_number_string(number(total)));
}
// Dates are no counters ("08-29-2011" read as 8 of 29).
let undated = replace(&DATES, &s, " ", true);
@@ -959,6 +960,11 @@ fn name_like(b: &[u8]) -> bool {
})
}
/// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal).
fn broken_total(index: &[u8], total: &[u8]) -> bool {
total.len() > 1 && total[0] == b'0' && total.len() > index.len()
}
/// The second number of the first \[(\d+)/(\d+)\].
fn file_counter_total(s: &str) -> Option<i64> {
let b = s.as_bytes();
@@ -973,6 +979,9 @@ fn file_counter_total(s: &str) -> Option<i64> {
}
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
if broken_total(&b[i + 1..slash], &b[slash + 1..slash + 1 + d]) {
return None;
}
return js::parse_int(&s[slash + 1..slash + 1 + d]);
}
}
@@ -1283,6 +1292,14 @@ mod image_set_tests {
assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key);
}
#[test]
fn totals_padded_wider_than_the_index_are_no_counters() {
let who = |i: u32, n: &str| release_identity("p", &format!(r#"Very Best Of The Who [{i:02}/{:03}] - "{n}" yEnc"#, i * 10), n);
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, "The Who - 02 - Anyway.mp3").key);
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").file_total, None);
assert_eq!(super::counter_total(r#"x [001/010] - "a.rar""#).as_deref(), Some("10"));
}
#[test]
fn dates_lead_no_numbered_items_and_repeated_names_go() {
let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n);
+10 -2
View File
@@ -193,10 +193,18 @@ export function imageSet(stem: string): string {
return /(?<!\d)0*[1-9]\d{4,}/.test(set) ? '' : set
}
/**
* A total a posting tool wrote wrong: zero-padded wider than the index ("[01/010]", "[02/020]", "[07/070]": each
* track of an album had a total of its own, so a release of its own; LOG.md 2026-10-12 (15)).
*/
function brokenTotal(index: string, total: string): boolean {
return total.length > 1 && total.startsWith('0') && total.length > index.length
}
export function counterTotal(subject: string): number | undefined {
const s = subject.replace(/"[^"]*"/g, ' ')
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
if (plain) return Number(plain[2] ?? plain[4])
if (plain) return brokenTotal(plain[1] ?? plain[3], plain[2] ?? plain[4]) ? undefined : Number(plain[2] ?? plain[4])
// Dates are no counters ("Avira AntiVir Rescue System 08-29-2011.rar", "… 27-02-2011", "18-08-2012"):
// read as "8 of 29", such a release showed a wrong file total and counted as incomplete.
const undated = s.replace(DATES, ' ')
@@ -263,7 +271,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
const poster = randomPoster(input.poster) ? '' : input.poster
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
// The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's.
const postTotal = counter ? Number.parseInt(counter[2], 10) : undefined
const postTotal = counter && !brokenTotal(counter[1], counter[2]) ? Number.parseInt(counter[2], 10) : undefined
const stem = fileStem(input.name)
if (input.weak) {
+7
View File
@@ -276,6 +276,13 @@ test.group('release / par2 names', () => {
assert.notEqual(vlad('185', 77, 'P8250001.JPG').key, vlad('234', 77, 'P5300085.JPG').key)
})
test('a total padded wider than its index is no counter', ({ assert }) => {
const who = (i: number, n: string) =>
releaseIdentity({ poster: 'p', subject: `Very Best Of The Who [${String(i).padStart(2, '0')}/${String(i * 10).padStart(3, '0')}] - "${n}" yEnc`, name: n, weak: false })!
assert.equal(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, 'The Who - 02 - Anyway.mp3').key)
assert.isUndefined(who(3, "The Who - 01 - I Can't Explain.mp3").fileTotal)
})
test('a leading date numbers no item; a file name repeated in the subject goes everywhere', ({ assert }) => {
const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })!
const show = (n: string) => id(`#sterntuary - Jeff Rense Radio Program - "${n}" yEnc`, n, '#sterntuary <aa@aa.com>')