From 625ea23171682a1e4c8b557be44b3cc4e3169d7c Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 11 Oct 2026 23:15:31 +0200 Subject: [PATCH] More single-file splits from the re-index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releases of one file whose counter announces many (910k of 3M after the sixth reset's first minutes), by cause: - An unquoted name with double spaces ("lara_bath 15of19 larabath_2.5.jpg", "Luisa: (068 of 160)"): the subject is collapsed, the name wasn't, so it stayed in the remainder; it's taken out collapsed too. - Audiobook and album tracks keyed by their own name ("RP - File 268 of 295 - \"… Seeing A Large Cat - D11.16-22.mp3\"", "CANDYD :: \"Candy Dulfer - Candy Store - 11 - everytime.mp3\""): by the name without its numbers, as image sets. - A size without a unit before the closing yEnc ("…mp3\" - 8123 yEnc"). - Random posters with separators ("SOFZRw1-OTEd@…", "…_OOW@…"). Both implementations; tests; fixtures identical (models: 267 -> 172). Co-Authored-By: Claude Opus 5.5 --- edge/src/names.rs | 72 +++++++++++++++++++++++++++-- lib/release/identity.ts | 19 +++++++- tests/unit/release/identity.spec.ts | 20 ++++++++ 3 files changed, 106 insertions(+), 5 deletions(-) diff --git a/edge/src/names.rs b/edge/src/names.rs index 49f97bf..4a23706 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -328,7 +328,32 @@ fn remove_counters_and_sizes(s: &str) -> String { let s = replace_counters(&SLASH_COUNTER, &s); let s = replace_counters(&ZERO_COUNTER, &s); let s = replace(&SIZES, &s, " ", true); - replace(&BRACKET_SIZES, &s, " ", true) + let s = replace(&BRACKET_SIZES, &s, " ", true); + strip_number_before_yenc(&s) +} + +/// `.replace(/\b\d+(\s+yEnc\s*)$/i, '$1')`: a size without a unit right before the closing yEnc. +fn strip_number_before_yenc(s: &str) -> String { + let tail_start = s.trim_end_matches(js::is_space).len(); + let body = &s[..tail_start]; + if body.len() < 4 || !body.is_char_boundary(body.len() - 4) || !body[body.len() - 4..].eq_ignore_ascii_case("yenc") { + return s.to_string(); + } + let before = &body[..body.len() - 4]; + let digits_end = before.trim_end_matches(js::is_space).len(); + if digits_end == before.len() { + return s.to_string(); // \s+ needs a space + } + let b = before.as_bytes(); + let digits_start = b[..digits_end].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1); + if digits_start == digits_end { + return s.to_string(); + } + // \b: the digit run starts the text or follows a non-word character. + if digits_start > 0 && (b[digits_start - 1].is_ascii_alphanumeric() || b[digits_start - 1] == b'_') { + return s.to_string(); + } + format!("{}{}", &s[..digits_start], &s[digits_end..]) } /// The end of subjectRemainder: what's left of brackets, double spaces, trim (after the separators became single spaces). @@ -517,7 +542,9 @@ pub fn random_poster(from: &str) -> bool { Some(address) => address, None => js::trim(from).split(' ').next().unwrap_or(""), }; - looks_random(address.split_once('@').map_or(address, |(local, _)| local)) + // Separators don't make a token less random. + let local: String = address.split_once('@').map_or(address, |(local, _)| local).chars().filter(|c| !matches!(c, '-' | '_' | '.')).collect(); + looks_random(&local) } pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { @@ -527,7 +554,15 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let file_total = counter_total(subject).and_then(|t| t.parse::().ok()); let stem = file_stem(name); // A name that isn't quoted (unquoted_name) is taken out as the quotes would have been. - let remainder = if subject.contains(&format!("\"{name}\"")) { subject_remainder(subject) } else { subject_remainder(&subject.replacen(name, " ", 1).replace('"', " ")) }; + // The subject here has its spaces collapsed; the name may not. + let remainder = if subject.contains(&format!("\"{name}\"")) { + subject_remainder(subject) + } else { + let collapsed = js::collapse_spaces(name); + let collapsed = js::trim(&collapsed); + let without = if subject.contains(name) { subject.replacen(name, " ", 1) } else { subject.replacen(collapsed, " ", 1) }; + subject_remainder(&without.replace('"', " ")) + }; // A document or comic is a release of its own. if test(&DOCUMENT, name) { let shown = replace(&DOCUMENT, &stem, "", false); @@ -559,6 +594,13 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let name = if prefix.is_empty() { stem.clone() } else { prefix.clone() }; return Release { key: hash(&["stem", poster, &name]), name, source: "stem", file_total }; } + // An audiobook's or album's tracks without a name around them: by their name without its numbers. + if test(&AUDIO, name) { + let tracks = image_prefix(&stem); + if !tracks.is_empty() { + return Release { key: hash(&["stem", poster, &tracks]), name: tracks, source: "stem", file_total }; + } + } Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total } } @@ -1056,6 +1098,30 @@ mod image_set_tests { assert!(!random_poster("w.merklein@freenet.de (muffin)")); } + #[test] + fn more_splits_from_the_reindex() { + use super::random_poster; + let makz = "Makz@BlueRadiance.uni (makz)"; + // The name as unquoted_name gives it (spaces kept), the subject collapsed and without its counter. + let a = release_identity(makz, r"G:\posting\thegoodnudes\lara_bath 15of19 larabath_2.5.jpg", r"G:\posting\thegoodnudes\lara_bath 15of19 larabath_2.5.jpg"); + let b = release_identity(makz, r"G:\posting\thegoodnudes\lara_bath 16of19 larabath_2.6.jpg", r"G:\posting\thegoodnudes\lara_bath 16of19 larabath_2.6.jpg"); + assert_eq!(a.key, b.key); + let mara = "Mara "; + let a = release_identity(mara, r#"RP - File 268 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3""#, "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3"); + let b = release_identity(mara, r#"RP - File 269 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3""#, "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3"); + assert_eq!(a.key, b.key); + let mb = "misterb@sure.com (MisterB)"; + let a = release_identity(mb, r#"Herb Ernst@256 - NMR -[07/17] "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3" - 8123 yEnc"#, "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3"); + let b = release_identity(mb, r#"Herb Ernst@256 - NMR -[08/17] "Herb Ernst - Dreamflight II - 06 - Other.mp3" - 9001 yEnc"#, "Herb Ernst - Dreamflight II - 06 - Other.mp3"); + assert_eq!(a.key, b.key); + for s in [r#"x - "a.mp3" - 8123 yEnc"#, "a8123 yEnc", "-8123 yEnc ", "yEnc"] { + assert_eq!(super::subject_remainder(s), super::subject_remainder_re(s), "{s}"); + } + assert!(random_poster("SOFZRw1-OTEd@EEFUbh.srG")); + assert!(random_poster("vMgGYlZB9ZcTNk8WxYFGV_OOW@x6YR.abc")); + assert!(!random_poster("w.merklein@freenet.de (muffin)")); + } + #[test] fn sizes_in_brackets_and_one_word_sets() { for s in [r#"Aletta Ocean - "100628-035.jpg" yEnc [347K]"#, r#"Corie - "2877-041.jpg" yEnc [290K]"#, "x - !!!blob2010-102katie009.jpg 336573 byt"] { diff --git a/lib/release/identity.ts b/lib/release/identity.ts index 6287346..e568f96 100644 --- a/lib/release/identity.ts +++ b/lib/release/identity.ts @@ -77,6 +77,8 @@ export function subjectRemainder(subject: string): string { // File sizes ("76.88 MB", "335555 Bytes", "336573 byt" cut off, "[347K]") differ per file. .replace(/\b\d+(?:[.,]\d+)?\s*(?:[kmgt]i?b|bytes?|byt)\b/gi, ' ') .replace(/\[\s*\d+(?:[.,]\d+)?\s*[kmgt]\s*\]/gi, ' ') + // A size without a unit right before the closing yEnc ("…mp3" - 8123 yEnc"). + .replace(/\b\d+(\s+yEnc\s*)$/i, '$1') // File sizes follow "yEnc" at the end; other numbers (years, post ids) stay. .replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ') .replace(/\byEnc\b/gi, ' ') @@ -195,7 +197,8 @@ export function randomPoster(from: string): boolean { const close = open === -1 ? -1 : from.indexOf('>', open + 1) const address = close !== -1 ? from.slice(open + 1, close) : from.trim().split(' ')[0] const at = address.indexOf('@') - return looksRandom(at === -1 ? address : address.slice(0, at)) + // Separators don't make a token less random ("SOFZRw1-OTEd", "vMgGYlZB9ZcTNk8WxYFGV_OOW"). + return looksRandom((at === -1 ? address : address.slice(0, at)).replace(/[-_.]/g, '')) } export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { @@ -216,7 +219,13 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". A name that isn't // quoted (unquotedName: "DVDFab 8.2.0.5.rar (006/332)", "[keira_s04_0037_l.jpg" yEnc] (2/2)") is taken out // as the quotes would have been: otherwise every file of a set would name a release of its own. - const remainder = subjectRemainder(input.subject.includes(`"${input.name}"`) ? input.subject : input.subject.replace(input.name, ' ').replace(/"/g, ' ')) + // The subject here has its spaces collapsed; the name may not ("lara_bath 15of19 larabath_2.5.jpg"). + const collapsed = input.name.replace(/\s+/g, ' ').trim() + const remainder = subjectRemainder( + input.subject.includes(`"${input.name}"`) + ? input.subject + : (input.subject.includes(input.name) ? input.subject.replace(input.name, ' ') : input.subject.replace(collapsed, ' ')).replace(/"/g, ' ') + ) // A document or comic is a release of its own (collections post hundreds under one subject). if (DOCUMENT.test(input.name)) return { key: hash('stem', poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' } const image = IMAGE.test(input.name) @@ -252,6 +261,12 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { } // An image without words around it: its set by its name (keyed as a stem, so "x.jpg" joins "x.part01.rar"). if (image) return { key: hash('stem', poster, prefix || stem), name: prefix || stem, fileTotal, source: 'stem' } + // An audiobook's or album's tracks without a name around them: by their name without its numbers, as an + // image set ("… Seeing A Large Cat - D11.16-22.mp3", "Candy Dulfer - Candy Store - 11 - everytime.mp3"). + if (AUDIO.test(input.name)) { + const tracks = stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() + if (tracks) return { key: hash('stem', poster, tracks), name: tracks, fileTotal, source: 'stem' } + } // The [i/M] total is informational, not part of the key: posters often // upload the par2 index without a counter, and it belongs to the same post. return { key: hash('stem', poster, stem), name: stem, fileTotal, source: 'stem' } diff --git a/tests/unit/release/identity.spec.ts b/tests/unit/release/identity.spec.ts index baf9104..afa7cbb 100644 --- a/tests/unit/release/identity.spec.ts +++ b/tests/unit/release/identity.spec.ts @@ -194,6 +194,26 @@ test.group('release / par2 names', () => { assert.equal(id(`${album} - 07 - Rascals - Sattva.mp3 (1/43)`).key, id(`${album} - 08 - Incredible String Band - Koeeoaddi There.mp3 (1/47)`).key) }) + test('more splits from the re-index: spaces in unquoted names, audio tracks, sizes before yEnc, posters with separators', async ({ assert }) => { + const { parseSubject } = await import('#lib/headers/index') + const { randomPoster } = await import('#lib/release/identity') + // As the edge worker: the subject without its counter and with collapsed spaces, the name as parsed. + const id = (poster: string, subject: string) => { + const name = parseSubject(subject)!.name + const base = subject.replace(/\(\d+\/\d+\)\s*$/, '').replace(/\s+/g, ' ').trim() + return releaseIdentity({ poster, subject: base, name, weak: false })! + } + const makz = 'Makz@BlueRadiance.uni (makz)' + assert.equal(id(makz, 'G:\\posting\\thegoodnudes\\lara_bath 15of19 larabath_2.5.jpg (1/1)').key, id(makz, 'G:\\posting\\thegoodnudes\\lara_bath 16of19 larabath_2.6.jpg (1/1)').key) + const mara = 'Mara ' + assert.equal(id(mara, 'RP - File 268 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3" (1/3)').key, id(mara, 'RP - File 269 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3" (1/3)').key) + const mb = 'misterb@sure.com (MisterB)' + assert.equal(id(mb, 'Herb Ernst@256 - NMR -[07/17] "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3" - 8123 yEnc (1/9)').key, id(mb, 'Herb Ernst@256 - NMR -[08/17] "Herb Ernst - Dreamflight II - 06 - Other.mp3" - 9001 yEnc (1/9)').key) + assert.isTrue(randomPoster('SOFZRw1-OTEd@EEFUbh.srG')) + assert.isTrue(randomPoster('vMgGYlZB9ZcTNk8WxYFGV_OOW@x6YR.abc')) + assert.isFalse(randomPoster('w.merklein@freenet.de (muffin)')) + }) + test('image sets: sizes in brackets and one-word set names', ({ assert }) => { const id = (subject: string, name: string) => releaseIdentity({ poster: 'p', subject, name, weak: false })! assert.equal(id('Aletta Ocean - "100628-035.jpg" yEnc [347K]', '100628-035.jpg').key, id('Aletta Ocean - "100628-036.jpg" yEnc [298K]', '100628-036.jpg').key)