Rebuild: a poster's constant tag no longer makes one release of all their uploads

Release 145637 held four albums: the poster puts '[KiLL.dA.FuCkIn.
EaStErRaBbIt]----' before every upload, and the text outside the quotes
(when it names a release) was the key with the poster. The same made
one release of a poster's albums under '(PowBeertje80 Posted)' (a post
id, with the same [i/M]), of masterposts' films, and of one volume of
every set in a DVD9 post.

Now, when the file's stem describes something itself (12+ letters and
digits, two words of 3+ letters) and the text outside the quotes shares
no word with it (4+ letters as part of the stem, 3-letter words and
numbers whole; boilerplate like www/net/posted not counted; no 5+ digit
per-post id), that text is taken for the poster's tag and the stem
says which release the file is. Albums of track files ('FTA … Ketchup'),
'DIE_ZEIT_35' with '35_China_dl', 'FTD769997' posts, scene names with
abbreviated file names ('grp-sm') and generic sidecars ('file_id.diz')
keep their grouping.

releases:regroup re-keys a sample of the catalog with the current rule:
on homelab's test data 145637 → 4 albums, 111566 → 17 albums of 27
files, the rest of the subject releases that change mostly lose an ad
image ('Flyer-download-2-Day.gif'); stem releases unchanged. Ported to
edge-bench-rs: check.ts still finds all captures identical (ath 2013:
16 → 76 releases on both sides).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/releases_regroup.ts

@@ -0,0 +1,61 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+import { releaseIdentity } from '#lib/release/index'
+
+/**
+ * What lib/release's identity makes of releases already in the catalog: a
+ * sample of releases (by source), their files decoded from the chunks and
+ * each given its release key again. Shows how many would now split, and
+ * into what: for checking an identity change on real data.
+ */
+export default class ReleasesRegroup extends BaseCommand {
+  static commandName = 'releases:regroup'
+  static description = 'Shows how the current release identity would regroup a sample of the catalog'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled per source', default: 2000 })
+  declare sample: number
+
+  @flags.array({ description: 'Release ids to show in full' })
+  declare show?: string[]
+
+  async run() {
+    for (const id of this.show ?? []) {
+      const groups = await this.regroup(Number(id))
+      this.logger.info(`${id}: ${groups.size} release(s) now`)
+      for (const [, g] of groups) this.logger.info(`  ${g.files} files: ${g.name} (${g.source})`)
+    }
+    if (this.show?.length) return
+    for (const source of ['subject', 'post', 'stem']) {
+      const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
+      let split = 0
+      let pieces = 0
+      const examples: string[] = []
+      for (const r of rows) {
+        const groups = await this.regroup(Number(r.id))
+        if (groups.size <= 1) continue
+        split++
+        pieces += groups.size
+        if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`)
+      }
+      this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`)
+      for (const e of examples) this.logger.info(`  ${e.slice(0, 260)}`)
+    }
+  }
+
+  private async regroup(id: number) {
+    const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+    const groups = new Map<string, { name: string; source: string; files: number }>()
+    for (const c of chunks) {
+      for (const f of decodeChunk(c.payload)) {
+        const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
+        const g = groups.get(r.key)
+        if (g) g.files++
+        else groups.set(r.key, { name: r.name, source: r.source, files: 1 })
+      }
+    }
+    return groups
+  }
+}

lib/release/identity.ts

@@ -1,5 +1,6 @@
 import { hash as digest } from 'node:crypto'
 import { looksLikeFilename } from '../headers/file_key.js'
+import { looksRandom } from './random.js'
 
 export interface ReleaseIdentity {
   key: string
@@ -106,6 +107,39 @@ function hash(...parts: (string | number | undefined)[]) {
   return digest('sha1', parts.map((p) => p ?? '').join('\n'))
 }
 
+/** A name that says what it is: 12+ letters and digits, two words of 3+ letters. */
+function describes(stem: string): boolean {
+  return stem.replace(/[^A-Za-z0-9]/g, '').length >= 12 && /[A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}/.test(stem)
+}
+
+/** Words of a subject that say nothing about what is posted: hosts, posting boilerplate. */
+const TAG_WORDS = new Set(['www', 'http', 'https', 'com', 'net', 'org', 'info', 'the', 'and', 'presents', 'posted', 'post', 'powered', 'yenc', 'part', 'rar', 'par2', 'nzb', 'voor', 'for', 'from'])
+
+/**
+ * Whether the subject's text outside the quotes is a poster's constant tag
+ * rather than about the file: it shares no word with the file's stem. A word
+ * of 4+ letters may be part of the stem ("ketchup" in "ketchup_dance"); a
+ * 3-letter word or a number ("35" in "35_China_dl") must be a whole word;
+ * boilerplate doesn't count. A number of 5+ digits is a per-post id
+ * ("FTD 769997", "(781442)"), so never a constant tag.
+ */
+export function isPosterTag(remainder: string, stem: string): boolean {
+  const words = stem.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
+  const compact = words.replace(/ /g, '')
+  const whole = new Set(words.split(' '))
+  // A per-post id, also inside a word ("FTD772544").
+  if (/\d{5,}/.test(remainder)) return false
+  for (const word of remainder.toLowerCase().split(/[^a-z0-9]+/)) {
+    if (/^\d+$/.test(word)) {
+      if (word.length >= 2 && whole.has(word)) return false
+      continue
+    }
+    if (word.length < 3 || TAG_WORDS.has(word)) continue
+    if (word.length >= 4 ? compact.includes(word) : whole.has(word)) return false
+  }
+  return true
+}
+
 /**
  * Decides which release (download) a file belongs to. Returns null when the
  * file cannot be grouped from its name alone (random extensionless names in
@@ -125,7 +159,13 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
 
   // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
   const remainder = subjectRemainder(input.subject)
-  if (namesRelease(remainder)) {
+  // Unless it's the poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)"),
+  // which would make one release of all their uploads: a readable file name that shares no word
+  // with it says which release the file is (LOG.md 2026-10-09 (8)).
+  // Only when the stem itself describes something ("Cephalgy.-.Weisses.Fleisch.2003", not a scene
+  // abbreviation like "grp-sm" under "Some.Movie.2008-GRP", nor "file_id.diz").
+  const tag = describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
+  if (!tag && namesRelease(remainder)) {
     return {
       key: hash('subject', input.poster, remainder),
       name: remainder,
@@ -135,7 +175,7 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
   }
   // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
   // [i/M] counter it groups the files of one post whose names are random.
-  if (fileTotal && isPostToken(remainder)) {
+  if (!tag && fileTotal && isPostToken(remainder)) {
     return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
   }
   // The [i/M] total is informational, not part of the key: posters often

tests/unit/release/identity.spec.ts

@@ -166,4 +166,34 @@ test.group('release / par2 names', () => {
     assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP')
     assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024')
   })
+  test("a poster's constant tag doesn't make one release of all their uploads", ({ assert }) => {
+    const id = (poster: string, subject: string, name: string) => releaseIdentity({ poster, subject, name, weak: false })!
+    // [KiLL…] tag on every upload: each album by its stem, its nfo included.
+    const kill = 'www.ggb.6x.to <darkness@earth.de>'
+    const a = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc', 'Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00')
+    const aNfo = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc', 'Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo')
+    const b = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc', 'Shnarph.-.Lautlos.2008[[g.6x.to]].r00')
+    assert.equal(a.key, aNfo.key)
+    assert.notEqual(a.key, b.key)
+    assert.equal(a.source, 'stem')
+    // A tag taken for a post id: "(PowBeertje80 Posted)" with the same [i/M] on every album.
+    const pow = (album: string) => id('pow', `(PowBeertje80 Posted) [01/026] - "${album}.part01.rar" yEnc`, `${album}.part01.rar`)
+    assert.notEqual(pow('Bravo Hits Vol.14 2CD -1996-').key, pow('Bravo Hits Vol.33 2CD -2001-').key)
+  })
+
+  test('the text outside the quotes still groups when it describes the files, or is a per-post id', ({ assert }) => {
+    const id = (subject: string, name: string) => releaseIdentity({ poster: 'p', subject, name, weak: false })!
+    // A word in common ("ketchup"): one album of many track files.
+    const t1 = id('!!!!! FTA www.ftaclub.net BodyCrew Presents VA-Ketchup_Dance2008 !!!!! [01/28] - "fta-01-a_track-2008.mp3" yEnc', 'fta-01-a_track-2008.mp3')
+    const t2 = id('!!!!! FTA www.ftaclub.net BodyCrew Presents VA-Ketchup_Dance2008 !!!!! [02/28] - "fta-02-coyote-ketchup_mix.mp3" yEnc', 'fta-02-coyote-ketchup_mix.mp3')
+    assert.equal(t1.key, t2.key)
+    assert.equal(t1.source, 'subject')
+    // A number in common ("35"), and a per-post id ("FTD769997").
+    assert.equal(id('[Audiofiles_DIE_ZEIT_35] [01/24] - "35_China_dl.mp3" yEnc', '35_China_dl.mp3').key, id('[Audiofiles_DIE_ZEIT_35] [02/24] - "35_Bamberg_dl.mp3" yEnc', '35_Bamberg_dl.mp3').key)
+    assert.equal(id('FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc', 'Prison.Break.S04E02.720p.part01.rar').key, id('FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc', 'Prison.Break.S04E02.720p.part.par2').key)
+    // A scene name with abbreviated file names, and a generically named sidecar.
+    const scene = id('Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc', 'grp-sm.part01.rar')
+    assert.equal(scene.name, 'Some Movie 2008 DVDRip XviD GRP')
+    assert.equal(id('Better.Jpeg.v2.0-DOA [02/10] - "file_id.diz" yEnc', 'file_id.diz').key, id('Better.Jpeg.v2.0-DOA [03/10] - "doa-bj20.zip" yEnc', 'doa-bj20.zip').key)
+  })
 })

tools/edge-bench-rs/src/names.rs

@@ -348,15 +348,76 @@ fn hash(parts: &[&str]) -> String {
     sha1_hex(&parts.join("\n"))
 }
 
+/// A name that says what it is: 12+ letters and digits, two words of 3+ letters (`describes`).
+fn describes(stem: &str) -> bool {
+    let b = stem.as_bytes();
+    if b.iter().filter(|c| c.is_ascii_alphanumeric()).count() < 12 {
+        return false;
+    }
+    // [A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}
+    let mut i = 0;
+    while i < b.len() {
+        let letters = b[i..].iter().take_while(|c| c.is_ascii_alphabetic()).count();
+        if letters == 0 {
+            i += 1;
+            continue;
+        }
+        let end = i + letters;
+        if letters >= 3 {
+            let gap = b[end..].iter().take_while(|c| !c.is_ascii_alphanumeric()).count();
+            if gap > 0 && b[end + gap..].iter().take_while(|c| c.is_ascii_alphabetic()).count() >= 3 {
+                return true;
+            }
+        }
+        i = end;
+    }
+    false
+}
+
+const TAG_WORDS: [&str; 21] = [
+    "www", "http", "https", "com", "net", "org", "info", "the", "and", "presents", "posted", "post", "powered", "yenc", "part", "rar", "par2", "nzb", "voor", "for", "from",
+];
+
+/// isPosterTag: the text outside the quotes shares no word with the file's stem.
+pub fn is_poster_tag(remainder: &str, stem: &str) -> bool {
+    let lower = stem.to_lowercase();
+    let words: Vec<&str> = lower.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())).filter(|w| !w.is_empty()).collect();
+    let compact: String = words.concat();
+    let whole: std::collections::HashSet<&str> = words.iter().copied().collect();
+    // A per-post id, also inside a word ("FTD772544").
+    let b = remainder.as_bytes();
+    if b.windows(5).any(|w| w.iter().all(u8::is_ascii_digit)) {
+        return false;
+    }
+    let rem = remainder.to_lowercase();
+    for word in rem.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())) {
+        if !word.is_empty() && word.bytes().all(|c| c.is_ascii_digit()) {
+            if word.len() >= 2 && whole.contains(word) {
+                return false;
+            }
+            continue;
+        }
+        if word.len() < 3 || TAG_WORDS.contains(&word) {
+            continue;
+        }
+        if if word.len() >= 4 { compact.contains(word) } else { whole.contains(word) } {
+            return false;
+        }
+    }
+    true
+}
+
 /// releaseIdentity for a named (not weak) file: never null.
 pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
     let file_total = file_counter_total(subject);
     let stem = file_stem(name);
     let remainder = subject_remainder(subject);
-    if names_release(&remainder) {
+    // The poster's constant tag outside the quotes: the file's stem says which upload it is.
+    let tag = describes(&stem) && !looks_random(&stem) && is_poster_tag(&remainder, &stem);
+    if !tag && names_release(&remainder) {
         return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total };
     }
-    if let Some(total) = file_total.filter(|&n| n != 0) {
+    if let Some(total) = file_total.filter(|&n| n != 0 && !tag) {
         if is_post_token(&remainder) {
             return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
         }
@@ -722,6 +783,24 @@ mod tests {
         }
     }
 
+    #[test]
+    fn poster_tags_as_in_the_typescript() {
+        let kill = "www.ggb.6x.to <darkness@earth.de>";
+        let a = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00");
+        let nfo = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo");
+        let b = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc"#, "Shnarph.-.Lautlos.2008[[g.6x.to]].r00");
+        assert_eq!(a.key, nfo.key);
+        assert_ne!(a.key, b.key);
+        assert_eq!(a.source, "stem");
+        let ftd1 = release_identity("p", r#"FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc"#, "Prison.Break.S04E02.720p.part01.rar");
+        let ftd2 = release_identity("p", r#"FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc"#, "Prison.Break.S04E02.720p.part.par2");
+        assert_eq!(ftd1.key, ftd2.key);
+        let scene = release_identity("p", r#"Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc"#, "grp-sm.part01.rar");
+        assert_eq!(scene.name, "Some Movie 2008 DVDRip XviD GRP");
+        assert!(is_poster_tag("PowBeertje80 Posted", "Bravo Hits Vol.14 2CD -1996-"));
+        assert!(!is_poster_tag("Audiofiles DIE ZEIT 35", "35_China_dl"));
+    }
+
     #[test]
     fn random_names() {
         assert!(looks_random("prvjr7hapd0i1ny14pz4"));