Counters: a total padded wider than its index is none; report diff on shared ranges
- "Very Best Of The Who [01/010]", "[02/020]", "[07/070]": a posting tool's
total of index×10, so every track had a total of its own and was a
release of its own. A total zero-padded wider than its index is no
counter ("[001/010]" still is).
- corpus_sweep.py report compares builds over the ranges both replayed (the
corpus grows between builds).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1a5035d988
commit
8e07478659
4 files changed
+52
-11
No files matched your search
+17
-8
@@ -113,13 +113,20 @@ def replay():
|
|||||||
shutil.rmtree(os.path.join(OUT, b))
|
shutil.rmtree(os.path.join(OUT, b))
|
||||||
|
|
||||||
|
|
||||||
def load(build):
|
def replayed(build):
|
||||||
"""Releases of one build's replay, merged by (group, key) as the catalog does."""
|
"""The ranges a build replayed, as paths relative to its directory."""
|
||||||
|
root = os.path.join(OUT, build)
|
||||||
|
return {os.path.relpath(os.path.join(d, f), root) for d, _, files in os.walk(root) for f in files if f.endswith('.jsonl')}
|
||||||
|
|
||||||
|
|
||||||
|
def load(build, only=None):
|
||||||
|
"""Releases of one build's replay (of the ranges in `only`, when given), merged by (group, key) as the catalog does."""
|
||||||
releases = {}
|
releases = {}
|
||||||
for dirpath, _, files in os.walk(os.path.join(OUT, build)):
|
for rel in sorted(replayed(build)):
|
||||||
for f in files:
|
if only is not None and rel not in only:
|
||||||
if not f.endswith('.jsonl'):
|
|
||||||
continue
|
continue
|
||||||
|
dirpath, f = os.path.split(os.path.join(OUT, build, rel))
|
||||||
|
if True:
|
||||||
for line in open(os.path.join(dirpath, f), encoding='utf-8', errors='replace'):
|
for line in open(os.path.join(dirpath, f), encoding='utf-8', errors='replace'):
|
||||||
r = json.loads(line)
|
r = json.loads(line)
|
||||||
k = (r['group'], r['key'])
|
k = (r['group'], r['key'])
|
||||||
@@ -182,10 +189,12 @@ def report():
|
|||||||
show(r, ' ')
|
show(r, ' ')
|
||||||
|
|
||||||
if len(builds) >= 2:
|
if len(builds) >= 2:
|
||||||
before = load(builds[-2][1])
|
# Over the ranges both replayed: the corpus grows between builds.
|
||||||
print(f'\n== change since build {builds[-2][1]}: {len(before):,} → {len(rel):,} releases')
|
both = replayed(builds[-2][1]) & replayed(build)
|
||||||
|
before, after = load(builds[-2][1], both), load(build, both)
|
||||||
|
print(f'\n== change since build {builds[-2][1]} on the {len(both)} ranges both replayed: {len(before):,} → {len(after):,} releases')
|
||||||
names = lambda rs: collections.Counter((r['group'], r['name']) for r in rs.values())
|
names = lambda rs: collections.Counter((r['group'], r['name']) for r in rs.values())
|
||||||
a, b = names(before), names(rel)
|
a, b = names(before), names(after)
|
||||||
moved = sorted(set(a) | set(b), key=lambda k: -abs(a[k] - b[k]))[:20]
|
moved = sorted(set(a) | set(b), key=lambda k: -abs(a[k] - b[k]))[:20]
|
||||||
for k in moved:
|
for k in moved:
|
||||||
if a[k] != b[k]:
|
if a[k] != b[k]:
|
||||||
|
|||||||
+18
-1
@@ -385,8 +385,9 @@ fn finish_remainder(s: &str) -> String {
|
|||||||
pub fn counter_total(subject: &str) -> Option<String> {
|
pub fn counter_total(subject: &str) -> Option<String> {
|
||||||
let s = replace("ED, subject, " ", true);
|
let s = replace("ED, subject, " ", true);
|
||||||
if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) {
|
if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) {
|
||||||
|
let index = c.get(1).or(c.get(3)).unwrap().as_bytes();
|
||||||
let total = c.get(2).or(c.get(4)).unwrap().as_bytes();
|
let total = c.get(2).or(c.get(4)).unwrap().as_bytes();
|
||||||
return Some(js_number_string(number(total)));
|
return (!broken_total(index, total)).then(|| js_number_string(number(total)));
|
||||||
}
|
}
|
||||||
// Dates are no counters ("08-29-2011" read as 8 of 29).
|
// Dates are no counters ("08-29-2011" read as 8 of 29).
|
||||||
let undated = replace(&DATES, &s, " ", true);
|
let undated = replace(&DATES, &s, " ", true);
|
||||||
@@ -959,6 +960,11 @@ fn name_like(b: &[u8]) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal).
|
||||||
|
fn broken_total(index: &[u8], total: &[u8]) -> bool {
|
||||||
|
total.len() > 1 && total[0] == b'0' && total.len() > index.len()
|
||||||
|
}
|
||||||
|
|
||||||
/// The second number of the first \[(\d+)/(\d+)\].
|
/// The second number of the first \[(\d+)/(\d+)\].
|
||||||
fn file_counter_total(s: &str) -> Option<i64> {
|
fn file_counter_total(s: &str) -> Option<i64> {
|
||||||
let b = s.as_bytes();
|
let b = s.as_bytes();
|
||||||
@@ -973,6 +979,9 @@ fn file_counter_total(s: &str) -> Option<i64> {
|
|||||||
}
|
}
|
||||||
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||||||
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
|
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
|
||||||
|
if broken_total(&b[i + 1..slash], &b[slash + 1..slash + 1 + d]) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
return js::parse_int(&s[slash + 1..slash + 1 + d]);
|
return js::parse_int(&s[slash + 1..slash + 1 + d]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1283,6 +1292,14 @@ mod image_set_tests {
|
|||||||
assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key);
|
assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn totals_padded_wider_than_the_index_are_no_counters() {
|
||||||
|
let who = |i: u32, n: &str| release_identity("p", &format!(r#"Very Best Of The Who [{i:02}/{:03}] - "{n}" yEnc"#, i * 10), n);
|
||||||
|
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, "The Who - 02 - Anyway.mp3").key);
|
||||||
|
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").file_total, None);
|
||||||
|
assert_eq!(super::counter_total(r#"x [001/010] - "a.rar""#).as_deref(), Some("10"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn dates_lead_no_numbered_items_and_repeated_names_go() {
|
fn dates_lead_no_numbered_items_and_repeated_names_go() {
|
||||||
let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n);
|
let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n);
|
||||||
|
|||||||
+10
-2
@@ -193,10 +193,18 @@ export function imageSet(stem: string): string {
|
|||||||
return /(?<!\d)0*[1-9]\d{4,}/.test(set) ? '' : set
|
return /(?<!\d)0*[1-9]\d{4,}/.test(set) ? '' : set
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A total a posting tool wrote wrong: zero-padded wider than the index ("[01/010]", "[02/020]", "[07/070]": each
|
||||||
|
* track of an album had a total of its own, so a release of its own; LOG.md 2026-10-12 (15)).
|
||||||
|
*/
|
||||||
|
function brokenTotal(index: string, total: string): boolean {
|
||||||
|
return total.length > 1 && total.startsWith('0') && total.length > index.length
|
||||||
|
}
|
||||||
|
|
||||||
export function counterTotal(subject: string): number | undefined {
|
export function counterTotal(subject: string): number | undefined {
|
||||||
const s = subject.replace(/"[^"]*"/g, ' ')
|
const s = subject.replace(/"[^"]*"/g, ' ')
|
||||||
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
|
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
|
||||||
if (plain) return Number(plain[2] ?? plain[4])
|
if (plain) return brokenTotal(plain[1] ?? plain[3], plain[2] ?? plain[4]) ? undefined : Number(plain[2] ?? plain[4])
|
||||||
// Dates are no counters ("Avira AntiVir Rescue System 08-29-2011.rar", "… 27-02-2011", "18-08-2012"):
|
// Dates are no counters ("Avira AntiVir Rescue System 08-29-2011.rar", "… 27-02-2011", "18-08-2012"):
|
||||||
// read as "8 of 29", such a release showed a wrong file total and counted as incomplete.
|
// read as "8 of 29", such a release showed a wrong file total and counted as incomplete.
|
||||||
const undated = s.replace(DATES, ' ')
|
const undated = s.replace(DATES, ' ')
|
||||||
@@ -263,7 +271,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
|||||||
const poster = randomPoster(input.poster) ? '' : input.poster
|
const poster = randomPoster(input.poster) ? '' : input.poster
|
||||||
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
||||||
// The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's.
|
// The post rule's key takes the [i/M] total (as when the catalog started); the release's file total any counter's.
|
||||||
const postTotal = counter ? Number.parseInt(counter[2], 10) : undefined
|
const postTotal = counter && !brokenTotal(counter[1], counter[2]) ? Number.parseInt(counter[2], 10) : undefined
|
||||||
const stem = fileStem(input.name)
|
const stem = fileStem(input.name)
|
||||||
|
|
||||||
if (input.weak) {
|
if (input.weak) {
|
||||||
|
|||||||
@@ -276,6 +276,13 @@ test.group('release / par2 names', () => {
|
|||||||
assert.notEqual(vlad('185', 77, 'P8250001.JPG').key, vlad('234', 77, 'P5300085.JPG').key)
|
assert.notEqual(vlad('185', 77, 'P8250001.JPG').key, vlad('234', 77, 'P5300085.JPG').key)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('a total padded wider than its index is no counter', ({ assert }) => {
|
||||||
|
const who = (i: number, n: string) =>
|
||||||
|
releaseIdentity({ poster: 'p', subject: `Very Best Of The Who [${String(i).padStart(2, '0')}/${String(i * 10).padStart(3, '0')}] - "${n}" yEnc`, name: n, weak: false })!
|
||||||
|
assert.equal(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, 'The Who - 02 - Anyway.mp3').key)
|
||||||
|
assert.isUndefined(who(3, "The Who - 01 - I Can't Explain.mp3").fileTotal)
|
||||||
|
})
|
||||||
|
|
||||||
test('a leading date numbers no item; a file name repeated in the subject goes everywhere', ({ assert }) => {
|
test('a leading date numbers no item; a file name repeated in the subject goes everywhere', ({ assert }) => {
|
||||||
const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })!
|
const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })!
|
||||||
const show = (n: string) => id(`#sterntuary - Jeff Rense Radio Program - "${n}" yEnc`, n, '#sterntuary <aa@aa.com>')
|
const show = (n: string) => id(`#sterntuary - Jeff Rense Radio Program - "${n}" yEnc`, n, '#sterntuary <aa@aa.com>')
|
||||||
|
|||||||
Reference in new issue
Block a user