From 1866e38c256c3bdeeca323ec0d83882e4a678678 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 12 Oct 2026 12:10:40 +0200 Subject: [PATCH] Identity: per-file hex ids, dated stems, season packs under a bare counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the corpus sweep (695 ranges): - A bracketed hex id per file ("[PRiVATE] Rosewood.S02E08… [1f1793f68d5321] [newzNZB] [01/22]") goes from the subject: nEwZ[NZB]'s 32,101 single files. - A stem that starts with a date is specific enough for the poster-tag rule ("09-30-2008_CTC.r00" under "#sterntuary - Coast To Coast Radio Show": still one release of 25,188 files, the stem being too short to "describe"). - Episodes under a bare counter ("[01/36] "Pokemon - S02E01 - Pallet Party Panic.mkv"") are one release per show, season and counter total. Co-Authored-By: Claude Opus 5.5 --- edge/src/names.rs | 52 +++++++++++++++++++++++------ lib/release/identity.ts | 18 ++++++++-- tests/unit/release/identity.spec.ts | 14 ++++++++ 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/edge/src/names.rs b/edge/src/names.rs index 905e28e..83f84ba 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -655,6 +655,8 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let named = without.replace('"', " "); // A file's size in brackets after its counter ("[26/32] - [4929] "…part24.rar""). let named = replace(&SIZE_AFTER_COUNTER, &named, "$1", true); + // A per-file hex id in brackets ("… [1f1793f68d5321] [newzNZB] [01/22]"). + let named = replace(&HEX_ID, &named, " ", true); let remainder = subject_remainder(&named); // Counted without the file name: "shyla-shy-09-0195.jpg hardcore …" is no "9 of 195". let total = counter_total(&named); @@ -678,20 +680,14 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let letters = b.iter().take_while(|c| c.is_ascii_uppercase()).count(); let code_digits = b[letters..].iter().take_while(|c| c.is_ascii_digit()).count(); let code = (1..=5).contains(&letters) && (2..=6).contains(&code_digits) && matches!(b.get(letters + code_digits), Some(b'-' | b'_' | b' ')); - // Not a date: /^\d{1,2}[-._]\d{1,2}[-._](?:19|20)\d{2}(?!\d)/ ("09-30-2008_BAK_JRS"). - let date = { - let run = |at: usize| b.get(at..).map_or(0, |s| s.iter().take_while(|c| c.is_ascii_digit()).count()); - let sep = |at: usize| matches!(b.get(at), Some(b'-' | b'.' | b'_')); - let d1 = run(0); - let d2 = if (1..=2).contains(&d1) && sep(d1) { run(d1 + 1) } else { 0 }; - let y = d1 + 1 + d2 + 1; - (1..=2).contains(&d2) && sep(d1 + 1 + d2) && run(y) == 4 && matches!(&b[y..y + 2], b"19" | b"20") - }; + let date = date_led(b); (index_at(0) || (code && index_at(letters + code_digits + 1))) && !date }; // A hashed stem under text without a counter is the upload, the text a tag ("Seit 1753 Tagen - usenet4ever.info - …"). let hashed = file_total.is_none() && test(&HASHED, &stem); - let tag = hashed || (!image && !numbered && !test(&AUDIO, name) && describes(&stem) && !looks_random(&stem) && is_poster_tag(&remainder, &stem)); + // A dated stem says which upload it is, short as it is ("09-30-2008_CTC.r00"). + let dated = date_led(stem.as_bytes()); + let tag = hashed || (!image && !numbered && !test(&AUDIO, name) && (describes(&stem) || dated) && !looks_random(&stem) && is_poster_tag(&remainder, &stem)); let prefix = if image { image_prefix(&stem) } else { String::new() }; // An image set's subject may be one word, the model's name: /^[A-Za-z]{3,}$/. let one_word = image && remainder.len() >= 3 && remainder.bytes().all(|c| c.is_ascii_alphabetic()); @@ -726,6 +722,16 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { return Release { key: hash(&["stem", poster, &tracks]), name: tracks, source: "stem", file_total }; } } + // A season's episodes under a bare counter ("Pokemon - S02E01 - Pallet Party Panic.mkv"): per show, season and total. + if let Some(total) = post_total.filter(|&n| n != 0) { + if test(&VIDEO, name) { + if let Some(c) = EPISODE.captures(stem.as_bytes()) { + let show = String::from_utf8(c[1].to_vec()).unwrap(); + let season = format!("{} S{:0>2}", js::trim(&show), std::str::from_utf8(&c[2]).unwrap()); + return Release { key: hash(&["stem", poster, &season.to_lowercase(), &total.to_string()]), name: season, source: "stem", file_total }; + } + } + } // A numbered volume as the extension, within the counter's total ("DN.VPL PCK.1508" of 2195): one upload. let base = match (VOLUME_NUMBER.captures(stem.as_bytes()), post_total.filter(|&n| n != 0)) { (Some(c), Some(total)) if number(&c[2]) <= total as f64 => String::from_utf8(c[1].to_vec()).unwrap(), @@ -739,6 +745,9 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { Release { key: hash(&["stem", poster, key_stem]), name: stem, source: "stem", file_total } } +static HEX_ID: LazyLock = LazyLock::new(|| re(r"(?i)\[[0-9a-f]{8,}\]")); +static VIDEO: LazyLock = LazyLock::new(|| re(r"(?i)\.(mkv|mp4|avi|m4v|ts|wmv|mov)$")); +static EPISODE: LazyLock = LazyLock::new(|| re(r"(?i)^(.+?)[ ._-]+S([0-9]{1,2})E[0-9]{1,3}")); static SIZE_AFTER_COUNTER: LazyLock = LazyLock::new(|| re(r"(\[[0-9]+/[0-9]+\])SP*-SP*\[[0-9]+\]")); static HASHED: LazyLock = LazyLock::new(|| re(r"(?i)^[0-9a-f]{8,}(?:_[0-9a-f]{6,}){2,}(?:-[0-9a-z]+)?$")); static SCENE_TRACK: LazyLock = LazyLock::new(|| re(r"^[0-9]{1,3}[-_ .]+(.+?)(?:_-_| - |-)")); @@ -969,6 +978,16 @@ fn name_like(b: &[u8]) -> bool { }) } +/// /^\d{1,2}[-._]\d{1,2}[-._](?:19|20)\d{2}(?!\d)/: a stem that starts with a date ("09-30-2008_BAK_JRS"). +fn date_led(b: &[u8]) -> bool { + let run = |at: usize| b.get(at..).map_or(0, |s| s.iter().take_while(|c| c.is_ascii_digit()).count()); + let sep = |at: usize| matches!(b.get(at), Some(b'-' | b'.' | b'_')); + let d1 = run(0); + let d2 = if (1..=2).contains(&d1) && sep(d1) { run(d1 + 1) } else { 0 }; + let y = d1 + 1 + d2 + 1; + (1..=2).contains(&d2) && sep(d1 + 1 + d2) && run(y) == 4 && matches!(&b[y..y + 2], b"19" | b"20") +} + /// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal). fn broken_total(index: &[u8], total: &[u8]) -> bool { total.len() > 1 && total[0] == b'0' && total.len() > index.len() @@ -1301,6 +1320,19 @@ mod image_set_tests { assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key); } + #[test] + fn hex_ids_dated_stems_and_season_packs() { + let nz = |id: &str, i: u32, n: &str| release_identity("nEwZ", &format!(r#"[PRiVATE] Rosewood.S02E08.SPANiSH.1080p.HDTV.x264-FCC [{id}] [newzNZB] [{i:02}/22] - "{n}" yEnc"#), n); + assert_eq!(nz("1f1793f68d5321", 1, "[05cbdb538a]_x.nfo").key, nz("65e6464bc03cad", 2, "[e8a4e2c182]_x.r03").key); + let show = |n: &str| release_identity("#sterntuary ", &format!(r#"#sterntuary - Coast To Coast Radio Show - "{n}" yEnc"#), n); + assert_eq!(show("09-30-2008_CTC.nfo").key, show("09-30-2008_CTC.r00").key); + assert_ne!(show("09-30-2008_CTC.r00").key, show("10-01-2008_CTC.r00").key); + let ep = |i: u32, n: &str| release_identity("p", &format!(r#"[{i:02}/36] "{n}""#), n); + assert_eq!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").key, ep(2, "Pokemon - S02E02 - Gotta Catch Ya Later.mkv").key); + assert_eq!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").name, "Pokemon S02"); + assert_ne!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").key, ep(1, "Pokemon - S03E01 - x.mkv").key); + } + #[test] fn scene_index_files_go_with_the_release() { let p = "SavageCobra "; diff --git a/lib/release/identity.ts b/lib/release/identity.ts index 1765b0c..4fc60fa 100644 --- a/lib/release/identity.ts +++ b/lib/release/identity.ts @@ -128,6 +128,9 @@ function hash(...parts: (string | number | undefined)[]) { return digest('sha1', parts.map((p) => p ?? '').join('\n')) } +const DATE_LED = /^\d{1,2}[-._]\d{1,2}[-._](?:19|20)\d{2}(?!\d)/ +const VIDEO = /\.(mkv|mp4|avi|m4v|ts|wmv|mov)$/i +const EPISODE = /^(.+?)[ ._-]+S(\d{1,2})E\d{1,3}/i const HASHED = /^[0-9a-f]{8,}(?:_[0-9a-f]{6,}){2,}(?:-[0-9a-z]+)?$/i const SCENE_TRACK = /^\d{1,3}[-_ .]+(.+?)(?:_-_| - |-)/ const VOLUME_NUMBER = /^(.*[^\d. ])[. ](\d{1,5})\.?$/ @@ -302,6 +305,8 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { .replace(/"/g, ' ') // A file's size in brackets after its counter ("[26/32] - [4929] "…part24.rar"": each file a release; Kingdn). .replace(/(\[\d+\/\d+\])\s*-\s*\[\d+\]/g, '$1') + // A per-file hex id in brackets ("[PRiVATE] Rosewood.S02E08… [1f1793f68d5321] [newzNZB] [01/22]": 32,101 single files). + .replace(/\[[0-9a-f]{8,}\]/gi, ' ') const remainder = subjectRemainder(named) // Counted without the file name, as quoted names are: "shyla-shy-09-0195.jpg hardcore …" is no "9 of 195" // (each of 866 images was a release of its own; LOG.md 2026-10-12 (5)). @@ -319,11 +324,13 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { // lessons were releases of their own; LOG.md 2026-10-11 (7)). Uploads hours apart are apart anyway. // Also after a disc code ("SC8367-12 - Wynette, Tammy - Singing My Song", karaoke discs). // Not a date ("09-30-2008_BAK_JRS.r00": a daily show's 32,621 files were one release under the poster's tag). - const numbered = /^(?:[A-Z]{1,5}\d{2,6}[-_ ])?\d{1,3}[ ._-]/.test(stem) && !/^\d{1,2}[-._]\d{1,2}[-._](?:19|20)\d{2}(?!\d)/.test(stem) + const dated = DATE_LED.test(stem) + const numbered = /^(?:[A-Z]{1,5}\d{2,6}[-_ ])?\d{1,3}[ ._-]/.test(stem) && !dated // A hashed stem under text without a counter is the upload, the text a tag ("Seit 1753 Tagen - usenet4ever.info - // "055631a1_b6c820bc_a856086c_85112a5-u4e.part01.rar"": a day's uploads were one release; LOG.md 2026-10-12 (11)). const hashed = fileTotal === undefined && HASHED.test(stem) - const tag = hashed || (!image && !numbered && !AUDIO.test(input.name) && describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)) + // A dated stem says which upload it is, short as it is ("09-30-2008_CTC.r00" under "#sterntuary - Coast To Coast Radio Show"). + const tag = hashed || (!image && !numbered && !AUDIO.test(input.name) && (describes(stem) || dated) && !looksRandom(stem) && isPosterTag(remainder, stem)) // An image set without a counter: the image's name up to its last number, without numbers // ("Kitchen Hottie 1 set233-76-lg" → "Kitchen Hottie set", "bm_jessica_blb_h087" → "bm jessica blb h"). const prefix = image ? stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() : '' @@ -362,6 +369,13 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { const tracks = stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() if (tracks) return { key: hash('stem', poster, tracks), name: tracks, fileTotal, source: 'stem' } } + // A season's episodes under a bare counter ("[01/36] "Pokemon - S02E01 - Pallet Party Panic.mkv""): one release per + // show, season and counter total (each episode was a release of its own). + const episode = postTotal && VIDEO.test(input.name) ? EPISODE.exec(stem) : null + if (episode) { + const season = `${episode[1].trim()} S${episode[2].padStart(2, '0')}` + return { key: hash('stem', poster, season.toLowerCase(), postTotal), name: season, fileTotal, source: 'stem' } + } // The [i/M] total is informational, not part of the key: posters often // upload the par2 index without a counter, and it belongs to the same post. // A numbered volume as the extension, within the counter's total ("(????) [0659/2195] - "DN.VPL PCK.1508"", diff --git a/tests/unit/release/identity.spec.ts b/tests/unit/release/identity.spec.ts index 1849b37..eb01ece 100644 --- a/tests/unit/release/identity.spec.ts +++ b/tests/unit/release/identity.spec.ts @@ -276,6 +276,20 @@ test.group('release / par2 names', () => { assert.notEqual(vlad('185', 77, 'P8250001.JPG').key, vlad('234', 77, 'P5300085.JPG').key) }) + test('per-file hex ids go; dated stems are specific; season packs under a bare counter', ({ assert }) => { + const id = (poster: string, subject: string, name: string) => releaseIdentity({ poster, subject, name, weak: false })! + const nz = (hex: string, i: number, n: string) => + id('nEwZ', `[PRiVATE] Rosewood.S02E08.SPANiSH.1080p.HDTV.x264-FCC [${hex}] [newzNZB] [${String(i).padStart(2, '0')}/22] - "${n}" yEnc`, n) + assert.equal(nz('1f1793f68d5321', 1, '[05cbdb538a]_x.nfo').key, nz('65e6464bc03cad', 2, '[e8a4e2c182]_x.r03').key) + const show = (n: string) => id('#sterntuary ', `#sterntuary - Coast To Coast Radio Show - "${n}" yEnc`, n) + assert.equal(show('09-30-2008_CTC.nfo').key, show('09-30-2008_CTC.r00').key) + assert.notEqual(show('09-30-2008_CTC.r00').key, show('10-01-2008_CTC.r00').key) + const ep = (i: number, n: string) => id('p', `[${String(i).padStart(2, '0')}/36] "${n}"`, n) + assert.equal(ep(1, 'Pokemon - S02E01 - Pallet Party Panic.mkv').key, ep(2, 'Pokemon - S02E02 - Gotta Catch Ya Later.mkv').key) + assert.equal(ep(1, 'Pokemon - S02E01 - Pallet Party Panic.mkv').name, 'Pokemon S02') + assert.notEqual(ep(1, 'Pokemon - S02E01 - Pallet Party Panic.mkv').key, ep(1, 'Pokemon - S03E01 - x.mkv').key) + }) + test("a scene index's files go with the release: no case, no 00-", ({ assert }) => { const id = (n: string) => releaseIdentity({ poster: 'SavageCobra ', subject: n, name: n, weak: false })! const a = id('00-laura_van_dam_-_capture_radio_113-cable-10-10-2026-talion.m3u')