Counters: a total padded wider than its index is none; report diff on shared ranges
- "Very Best Of The Who [01/010]", "[02/020]", "[07/070]": a posting tool's
total of index×10, so every track had a total of its own and was a
release of its own. A total zero-padded wider than its index is no
counter ("[001/010]" still is).
- corpus_sweep.py report compares builds over the ranges both replayed (the
corpus grows between builds).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1a5035d988
commit
8e07478659
4 files changed
+53
-12
No files matched your search
+18
-9
@@ -113,13 +113,20 @@ def replay():
|
||||
shutil.rmtree(os.path.join(OUT, b))
|
||||
|
||||
|
||||
def load(build):
|
||||
"""Releases of one build's replay, merged by (group, key) as the catalog does."""
|
||||
def replayed(build):
|
||||
"""The ranges a build replayed, as paths relative to its directory."""
|
||||
root = os.path.join(OUT, build)
|
||||
return {os.path.relpath(os.path.join(d, f), root) for d, _, files in os.walk(root) for f in files if f.endswith('.jsonl')}
|
||||
|
||||
|
||||
def load(build, only=None):
|
||||
"""Releases of one build's replay (of the ranges in `only`, when given), merged by (group, key) as the catalog does."""
|
||||
releases = {}
|
||||
for dirpath, _, files in os.walk(os.path.join(OUT, build)):
|
||||
for f in files:
|
||||
if not f.endswith('.jsonl'):
|
||||
continue
|
||||
for rel in sorted(replayed(build)):
|
||||
if only is not None and rel not in only:
|
||||
continue
|
||||
dirpath, f = os.path.split(os.path.join(OUT, build, rel))
|
||||
if True:
|
||||
for line in open(os.path.join(dirpath, f), encoding='utf-8', errors='replace'):
|
||||
r = json.loads(line)
|
||||
k = (r['group'], r['key'])
|
||||
@@ -182,10 +189,12 @@ def report():
|
||||
show(r, ' ')
|
||||
|
||||
if len(builds) >= 2:
|
||||
before = load(builds[-2][1])
|
||||
print(f'\n== change since build {builds[-2][1]}: {len(before):,} → {len(rel):,} releases')
|
||||
# Over the ranges both replayed: the corpus grows between builds.
|
||||
both = replayed(builds[-2][1]) & replayed(build)
|
||||
before, after = load(builds[-2][1], both), load(build, both)
|
||||
print(f'\n== change since build {builds[-2][1]} on the {len(both)} ranges both replayed: {len(before):,} → {len(after):,} releases')
|
||||
names = lambda rs: collections.Counter((r['group'], r['name']) for r in rs.values())
|
||||
a, b = names(before), names(rel)
|
||||
a, b = names(before), names(after)
|
||||
moved = sorted(set(a) | set(b), key=lambda k: -abs(a[k] - b[k]))[:20]
|
||||
for k in moved:
|
||||
if a[k] != b[k]:
|
||||
|
||||
+18
-1
@@ -385,8 +385,9 @@ fn finish_remainder(s: &str) -> String {
|
||||
pub fn counter_total(subject: &str) -> Option<String> {
|
||||
let s = replace("ED, subject, " ", true);
|
||||
if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) {
|
||||
let index = c.get(1).or(c.get(3)).unwrap().as_bytes();
|
||||
let total = c.get(2).or(c.get(4)).unwrap().as_bytes();
|
||||
return Some(js_number_string(number(total)));
|
||||
return (!broken_total(index, total)).then(|| js_number_string(number(total)));
|
||||
}
|
||||
// Dates are no counters ("08-29-2011" read as 8 of 29).
|
||||
let undated = replace(&DATES, &s, " ", true);
|
||||
@@ -959,6 +960,11 @@ fn name_like(b: &[u8]) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal).
|
||||
fn broken_total(index: &[u8], total: &[u8]) -> bool {
|
||||
total.len() > 1 && total[0] == b'0' && total.len() > index.len()
|
||||
}
|
||||
|
||||
/// The second number of the first \[(\d+)/(\d+)\].
|
||||
fn file_counter_total(s: &str) -> Option<i64> {
|
||||
let b = s.as_bytes();
|
||||
@@ -973,6 +979,9 @@ fn file_counter_total(s: &str) -> Option<i64> {
|
||||
}
|
||||
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
|
||||
if broken_total(&b[i + 1..slash], &b[slash + 1..slash + 1 + d]) {
|
||||
return None;
|
||||
}
|
||||
return js::parse_int(&s[slash + 1..slash + 1 + d]);
|
||||
}
|
||||
}
|
||||
@@ -1283,6 +1292,14 @@ mod image_set_tests {
|
||||
assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn totals_padded_wider_than_the_index_are_no_counters() {
|
||||
let who = |i: u32, n: &str| release_identity("p", &format!(r#"Very Best Of The Who [{i:02}/{:03}] - "{n}" yEnc"#, i * 10), n);
|
||||
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, "The Who - 02 - Anyway.mp3").key);
|
||||
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").file_total, None);
|
||||
assert_eq!(super::counter_total(r#"x [001/010] - "a.rar""#).as_deref(), Some("10"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dates_lead_no_numbered_items_and_repeated_names_go() {
|
||||
let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n);
|
||||
|
||||
+10
-2
@@ -193,10 +193,18 @@ export function imageSet(stem: string): string {
|
||||
return /(?<!\d)0*[1-9]\d{4,}/.test(set) ? '' : set
|
||||
}
|
||||
|
||||
/**
|
||||
* A total a posting tool wrote wrong: zero-padded wider than the index ("[01/010]", "[02/020]", "[07/070]": each
|
||||
* track of an album had a total of its own, so a release of its own; LOG.md 2026-10-12 (15)).
|
||||
*/
|
||||
function brokenTotal(index: string, total: string): boolean {
|
||||
return total.length > 1 && total.startsWith('0') && total.length > index.length
|
||||
}
|
||||
|
||||
export function counterTotal(subject: string): number | undefined {
|
||||
const s = subject.replace(/"[^"]*"/g, ' ')
|
||||
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
|
||||
if (plain) return Number(plain[2] ?? plain[4])
|
||||
if (plain) return brokenTotal(plain[1] ?? plain[3], plain[2] ?? plain[4]) ? undefined : Number(plain[2] ?? plain[4])
|
||||
// Dates are no counters ("Avira AntiVir Rescue System 08-29-2011.rar", "… 27-02-2011", "18-08-2012"):
|
||||
// read as "8 of 29", such a release showed a wrong file total and counted as incomplete.
|
||||
const undated = s.replace(DATES, ' ')
|
||||
@@ -263,7 +271,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
const poster = randomPoster(input.poster) ? '' : input.poster
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
||||
// The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's.
|
||||
const postTotal = counter ? Number.parseInt(counter[2], 10) : undefined
|
||||
const postTotal = counter && !brokenTotal(counter[1], counter[2]) ? Number.parseInt(counter[2], 10) : undefined
|
||||
const stem = fileStem(input.name)
|
||||
|
||||
if (input.weak) {
|
||||
|
||||
@@ -276,6 +276,13 @@ test.group('release / par2 names', () => {
|
||||
assert.notEqual(vlad('185', 77, 'P8250001.JPG').key, vlad('234', 77, 'P5300085.JPG').key)
|
||||
})
|
||||
|
||||
test('a total padded wider than its index is no counter', ({ assert }) => {
|
||||
const who = (i: number, n: string) =>
|
||||
releaseIdentity({ poster: 'p', subject: `Very Best Of The Who [${String(i).padStart(2, '0')}/${String(i * 10).padStart(3, '0')}] - "${n}" yEnc`, name: n, weak: false })!
|
||||
assert.equal(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, 'The Who - 02 - Anyway.mp3').key)
|
||||
assert.isUndefined(who(3, "The Who - 01 - I Can't Explain.mp3").fileTotal)
|
||||
})
|
||||
|
||||
test('a leading date numbers no item; a file name repeated in the subject goes everywhere', ({ assert }) => {
|
||||
const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })!
|
||||
const show = (n: string) => id(`#sterntuary - Jeff Rense Radio Program - "${n}" yEnc`, n, '#sterntuary <aa@aa.com>')
|
||||
|
||||
Reference in new issue
Block a user