diff --git a/commands/releases_regroup.ts b/commands/releases_regroup.ts new file mode 100644 index 0000000..887825b --- /dev/null +++ b/commands/releases_regroup.ts @@ -0,0 +1,61 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import db from '@adonisjs/lucid/services/db' +import { decodeChunk } from '#lib/chunks/index' +import { releaseIdentity } from '#lib/release/index' + +/** + * What lib/release's identity makes of releases already in the catalog: a + * sample of releases (by source), their files decoded from the chunks and + * each given its release key again. Shows how many would now split, and + * into what: for checking an identity change on real data. + */ +export default class ReleasesRegroup extends BaseCommand { + static commandName = 'releases:regroup' + static description = 'Shows how the current release identity would regroup a sample of the catalog' + static options: CommandOptions = { startApp: true } + + @flags.number({ description: 'Releases sampled per source', default: 2000 }) + declare sample: number + + @flags.array({ description: 'Release ids to show in full' }) + declare show?: string[] + + async run() { + for (const id of this.show ?? []) { + const groups = await this.regroup(Number(id)) + this.logger.info(`${id}: ${groups.size} release(s) now`) + for (const [, g] of groups) this.logger.info(` ${g.files} files: ${g.name} (${g.source})`) + } + if (this.show?.length) return + for (const source of ['subject', 'post', 'stem']) { + const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files') + let split = 0 + let pieces = 0 + const examples: string[] = [] + for (const r of rows) { + const groups = await this.regroup(Number(r.id)) + if (groups.size <= 1) continue + split++ + pieces += groups.size + if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`) + } + this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`) + for (const e of examples) this.logger.info(` ${e.slice(0, 260)}`) + } + } + + private async regroup(id: number) { + const chunks = await db.from('release_chunks').where('release_id', id).select('payload') + const groups = new Map() + for (const c of chunks) { + for (const f of decodeChunk(c.payload)) { + const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })! + const g = groups.get(r.key) + if (g) g.files++ + else groups.set(r.key, { name: r.name, source: r.source, files: 1 }) + } + } + return groups + } +} diff --git a/lib/release/identity.ts b/lib/release/identity.ts index 98faf88..99aaa96 100644 --- a/lib/release/identity.ts +++ b/lib/release/identity.ts @@ -1,5 +1,6 @@ import { hash as digest } from 'node:crypto' import { looksLikeFilename } from '../headers/file_key.js' +import { looksRandom } from './random.js' export interface ReleaseIdentity { key: string @@ -106,6 +107,39 @@ function hash(...parts: (string | number | undefined)[]) { return digest('sha1', parts.map((p) => p ?? '').join('\n')) } +/** A name that says what it is: 12+ letters and digits, two words of 3+ letters. */ +function describes(stem: string): boolean { + return stem.replace(/[^A-Za-z0-9]/g, '').length >= 12 && /[A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}/.test(stem) +} + +/** Words of a subject that say nothing about what is posted: hosts, posting boilerplate. */ +const TAG_WORDS = new Set(['www', 'http', 'https', 'com', 'net', 'org', 'info', 'the', 'and', 'presents', 'posted', 'post', 'powered', 'yenc', 'part', 'rar', 'par2', 'nzb', 'voor', 'for', 'from']) + +/** + * Whether the subject's text outside the quotes is a poster's constant tag + * rather than about the file: it shares no word with the file's stem. A word + * of 4+ letters may be part of the stem ("ketchup" in "ketchup_dance"); a + * 3-letter word or a number ("35" in "35_China_dl") must be a whole word; + * boilerplate doesn't count. A number of 5+ digits is a per-post id + * ("FTD 769997", "(781442)"), so never a constant tag. + */ +export function isPosterTag(remainder: string, stem: string): boolean { + const words = stem.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim() + const compact = words.replace(/ /g, '') + const whole = new Set(words.split(' ')) + // A per-post id, also inside a word ("FTD772544"). + if (/\d{5,}/.test(remainder)) return false + for (const word of remainder.toLowerCase().split(/[^a-z0-9]+/)) { + if (/^\d+$/.test(word)) { + if (word.length >= 2 && whole.has(word)) return false + continue + } + if (word.length < 3 || TAG_WORDS.has(word)) continue + if (word.length >= 4 ? compact.includes(word) : whole.has(word)) return false + } + return true +} + /** * Decides which release (download) a file belongs to. Returns null when the * file cannot be grouped from its name alone (random extensionless names in @@ -125,7 +159,13 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". const remainder = subjectRemainder(input.subject) - if (namesRelease(remainder)) { + // Unless it's the poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)"), + // which would make one release of all their uploads: a readable file name that shares no word + // with it says which release the file is (LOG.md 2026-10-09 (8)). + // Only when the stem itself describes something ("Cephalgy.-.Weisses.Fleisch.2003", not a scene + // abbreviation like "grp-sm" under "Some.Movie.2008-GRP", nor "file_id.diz"). + const tag = describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem) + if (!tag && namesRelease(remainder)) { return { key: hash('subject', input.poster, remainder), name: remainder, @@ -135,7 +175,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { } // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the // [i/M] counter it groups the files of one post whose names are random. - if (fileTotal && isPostToken(remainder)) { + if (!tag && fileTotal && isPostToken(remainder)) { return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' } } // The [i/M] total is informational, not part of the key: posters often diff --git a/tests/unit/release/identity.spec.ts b/tests/unit/release/identity.spec.ts index 636fb06..4b50b1c 100644 --- a/tests/unit/release/identity.spec.ts +++ b/tests/unit/release/identity.spec.ts @@ -166,4 +166,34 @@ test.group('release / par2 names', () => { assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP') assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024') }) + test("a poster's constant tag doesn't make one release of all their uploads", ({ assert }) => { + const id = (poster: string, subject: string, name: string) => releaseIdentity({ poster, subject, name, weak: false })! + // [KiLL…] tag on every upload: each album by its stem, its nfo included. + const kill = 'www.ggb.6x.to ' + const a = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc', 'Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00') + const aNfo = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc', 'Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo') + const b = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc', 'Shnarph.-.Lautlos.2008[[g.6x.to]].r00') + assert.equal(a.key, aNfo.key) + assert.notEqual(a.key, b.key) + assert.equal(a.source, 'stem') + // A tag taken for a post id: "(PowBeertje80 Posted)" with the same [i/M] on every album. + const pow = (album: string) => id('pow', `(PowBeertje80 Posted) [01/026] - "${album}.part01.rar" yEnc`, `${album}.part01.rar`) + assert.notEqual(pow('Bravo Hits Vol.14 2CD -1996-').key, pow('Bravo Hits Vol.33 2CD -2001-').key) + }) + + test('the text outside the quotes still groups when it describes the files, or is a per-post id', ({ assert }) => { + const id = (subject: string, name: string) => releaseIdentity({ poster: 'p', subject, name, weak: false })! + // A word in common ("ketchup"): one album of many track files. + const t1 = id('!!!!! FTA www.ftaclub.net BodyCrew Presents VA-Ketchup_Dance2008 !!!!! [01/28] - "fta-01-a_track-2008.mp3" yEnc', 'fta-01-a_track-2008.mp3') + const t2 = id('!!!!! FTA www.ftaclub.net BodyCrew Presents VA-Ketchup_Dance2008 !!!!! [02/28] - "fta-02-coyote-ketchup_mix.mp3" yEnc', 'fta-02-coyote-ketchup_mix.mp3') + assert.equal(t1.key, t2.key) + assert.equal(t1.source, 'subject') + // A number in common ("35"), and a per-post id ("FTD769997"). + assert.equal(id('[Audiofiles_DIE_ZEIT_35] [01/24] - "35_China_dl.mp3" yEnc', '35_China_dl.mp3').key, id('[Audiofiles_DIE_ZEIT_35] [02/24] - "35_Bamberg_dl.mp3" yEnc', '35_Bamberg_dl.mp3').key) + assert.equal(id('FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc', 'Prison.Break.S04E02.720p.part01.rar').key, id('FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc', 'Prison.Break.S04E02.720p.part.par2').key) + // A scene name with abbreviated file names, and a generically named sidecar. + const scene = id('Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc', 'grp-sm.part01.rar') + assert.equal(scene.name, 'Some Movie 2008 DVDRip XviD GRP') + assert.equal(id('Better.Jpeg.v2.0-DOA [02/10] - "file_id.diz" yEnc', 'file_id.diz').key, id('Better.Jpeg.v2.0-DOA [03/10] - "doa-bj20.zip" yEnc', 'doa-bj20.zip').key) + }) }) diff --git a/tools/edge-bench-rs/src/names.rs b/tools/edge-bench-rs/src/names.rs index 5be17ea..7a44704 100644 --- a/tools/edge-bench-rs/src/names.rs +++ b/tools/edge-bench-rs/src/names.rs @@ -348,15 +348,76 @@ fn hash(parts: &[&str]) -> String { sha1_hex(&parts.join("\n")) } +/// A name that says what it is: 12+ letters and digits, two words of 3+ letters (`describes`). +fn describes(stem: &str) -> bool { + let b = stem.as_bytes(); + if b.iter().filter(|c| c.is_ascii_alphanumeric()).count() < 12 { + return false; + } + // [A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,} + let mut i = 0; + while i < b.len() { + let letters = b[i..].iter().take_while(|c| c.is_ascii_alphabetic()).count(); + if letters == 0 { + i += 1; + continue; + } + let end = i + letters; + if letters >= 3 { + let gap = b[end..].iter().take_while(|c| !c.is_ascii_alphanumeric()).count(); + if gap > 0 && b[end + gap..].iter().take_while(|c| c.is_ascii_alphabetic()).count() >= 3 { + return true; + } + } + i = end; + } + false +} + +const TAG_WORDS: [&str; 21] = [ + "www", "http", "https", "com", "net", "org", "info", "the", "and", "presents", "posted", "post", "powered", "yenc", "part", "rar", "par2", "nzb", "voor", "for", "from", +]; + +/// isPosterTag: the text outside the quotes shares no word with the file's stem. +pub fn is_poster_tag(remainder: &str, stem: &str) -> bool { + let lower = stem.to_lowercase(); + let words: Vec<&str> = lower.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())).filter(|w| !w.is_empty()).collect(); + let compact: String = words.concat(); + let whole: std::collections::HashSet<&str> = words.iter().copied().collect(); + // A per-post id, also inside a word ("FTD772544"). + let b = remainder.as_bytes(); + if b.windows(5).any(|w| w.iter().all(u8::is_ascii_digit)) { + return false; + } + let rem = remainder.to_lowercase(); + for word in rem.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())) { + if !word.is_empty() && word.bytes().all(|c| c.is_ascii_digit()) { + if word.len() >= 2 && whole.contains(word) { + return false; + } + continue; + } + if word.len() < 3 || TAG_WORDS.contains(&word) { + continue; + } + if if word.len() >= 4 { compact.contains(word) } else { whole.contains(word) } { + return false; + } + } + true +} + /// releaseIdentity for a named (not weak) file: never null. pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let file_total = file_counter_total(subject); let stem = file_stem(name); let remainder = subject_remainder(subject); - if names_release(&remainder) { + // The poster's constant tag outside the quotes: the file's stem says which upload it is. + let tag = describes(&stem) && !looks_random(&stem) && is_poster_tag(&remainder, &stem); + if !tag && names_release(&remainder) { return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total }; } - if let Some(total) = file_total.filter(|&n| n != 0) { + if let Some(total) = file_total.filter(|&n| n != 0 && !tag) { if is_post_token(&remainder) { return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total }; } @@ -722,6 +783,24 @@ mod tests { } } + #[test] + fn poster_tags_as_in_the_typescript() { + let kill = "www.ggb.6x.to "; + let a = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00"); + let nfo = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo"); + let b = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc"#, "Shnarph.-.Lautlos.2008[[g.6x.to]].r00"); + assert_eq!(a.key, nfo.key); + assert_ne!(a.key, b.key); + assert_eq!(a.source, "stem"); + let ftd1 = release_identity("p", r#"FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc"#, "Prison.Break.S04E02.720p.part01.rar"); + let ftd2 = release_identity("p", r#"FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc"#, "Prison.Break.S04E02.720p.part.par2"); + assert_eq!(ftd1.key, ftd2.key); + let scene = release_identity("p", r#"Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc"#, "grp-sm.part01.rar"); + assert_eq!(scene.name, "Some Movie 2008 DVDRip XviD GRP"); + assert!(is_poster_tag("PowBeertje80 Posted", "Bravo Hits Vol.14 2CD -1996-")); + assert!(!is_poster_tag("Audiofiles DIE ZEIT 35", "35_China_dl")); + } + #[test] fn random_names() { assert!(looks_random("prvjr7hapd0i1ny14pz4"));