Rebuild: image sets, documents and audio albums as releases
Test and deploy / test (push) Successful in 22s
Test and deploy / deploy (push) Skipped

Picture groups made a release per image: 501k single-file releases in
the first 24 ranges of the bottom 500 groups (a.b.models 385k), ~120M
for all of them. Counters written out ('File 027 of 132', '[12of42]',
'104 of 472', '[2239-6389]', '02/17', '0097-1574'; only when the first
number is at most the second, so '[1948-12]' stays) and per-file sizes
('76.88 MB', '335555 Bytes') kept every subject unique.

- subjectRemainder removes those counters and sizes, and empty brackets.
- A subject-named release is bounded by its counter's total
  (counterTotal: the first form found): uploads a poster names alike
  stay apart, an upload's files (par2, nfo) together.
- Images and audio tracks don't take the poster-tag rule: the subject
  names the set or album. An image set without a counter: the image's
  name up to its last number, without numbers.
- Documents and comics (pdf, epub, mobi, azw3, djvu, cbr, cbz, cb7, chm,
  lit, doc, docx) are a release each: collections post hundreds under
  one subject ('cw-horror&mystery').

releases:regroup --group re-keys a group's indexed releases together:
cheerleaders 55,741 -> 3,165 releases, models 385,298 -> 40,313, German
audiobooks 4,106 -> 1,724, playstation 3,309 -> 2,345; comics and
e-books a release per item; boneless unchanged. Ported to edge-bench-rs;
check.ts: identical on all captures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 12:21:00 +02:00
1 parent 59481037fb
commit 14758a5b7a
4 files changed
+272 -15

No files matched your search

+57 -7
View File
@@ -61,14 +61,28 @@ export function fileStem(name: string): string {
* What remains of a subject once the filename, counters, "yEnc" and sizes are
* removed. Posters that name the release outside the quotes put it here.
*/
/** A file counter's text when its first number is at most its second, else the text itself (a date like [1948-12] stays). */
const counter = (text: string, i: string, n: string) => (Number(i) <= Number(n) ? ' ' : text)
export function subjectRemainder(subject: string): string {
return subject
.replace(/"[^"]*"/g, ' ')
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
// Counters written out: "File 027 of 132", "[21 of 25]", "104 of 472", "[2239-6389]", "02/17",
// "0097-1574" (each image of a set was a release of its own; LOG.md 2026-10-09 (9)).
.replace(/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b[\])]?/gi, counter)
.replace(/\[(\d+)-(\d+)\]/g, counter)
.replace(/\b(\d+)\/(\d+)\b/g, counter)
.replace(/\b(0\d*)-(\d+)\b/g, counter)
// File sizes ("76.88 MB", "335555 Bytes") differ per file.
.replace(/\b\d+(?:[.,]\d+)?\s*(?:[kmgt]i?b|bytes?)\b/gi, ' ')
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
.replace(/\byEnc\b/gi, ' ')
.replace(/[\s\-–.:|]+/g, ' ')
// What's left of "["name.jpg" yEnc]" and of counters and sizes in brackets.
.replace(/ ?(?:\[ ?\]|\( ?\)) ?/g, ' ')
.replace(/ {2,}/g, ' ')
.trim()
}
@@ -107,6 +121,30 @@ function hash(...parts: (string | number | undefined)[]) {
return digest('sha1', parts.map((p) => p ?? '').join('\n'))
}
/** Image files: a set's images are named one by one ("IMG_3086.jpg"); its subject names the set. */
const IMAGE = /\.(jpe?g|png|gif|bmp|webp|tiff?)$/i
/** Audio tracks: parts of an album or audiobook that the subject names ("072 - Die Teufelsuhr.MP3 - by DeckaR"). */
const AUDIO = /\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])/i
/** Documents and comics: each file is an item of its own, whatever the subject says ("cw-misc [020/109] "Action Comics 710.cbr""). */
const DOCUMENT = /\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$/i
/**
* The total of the subject's file counter, in the forms subjectRemainder
* removes, the first form found first: [i/M] or (i/M), "i of M", [i-M], i/M,
* 0i-M (the last four only when i <= M).
*/
export function counterTotal(subject: string): number | undefined {
const s = subject.replace(/"[^"]*"/g, ' ')
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
if (plain) return Number(plain[2] ?? plain[4])
for (const pattern of [/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b/gi, /\[(\d+)-(\d+)\]/g, /\b(\d+)\/(\d+)\b/g, /\b(0\d*)-(\d+)\b/g]) {
for (const m of s.matchAll(pattern)) if (Number(m[1]) <= Number(m[2])) return Number(m[2])
}
return undefined
}
/** A name that says what it is: 12+ letters and digits, two words of 3+ letters. */
function describes(stem: string): boolean {
return stem.replace(/[^A-Za-z0-9]/g, '').length >= 12 && /[A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}/.test(stem)
@@ -159,15 +197,25 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
const remainder = subjectRemainder(input.subject)
// Unless it's the poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)"),
// which would make one release of all their uploads: a readable file name that shares no word
// with it says which release the file is (LOG.md 2026-10-09 (8)).
// Only when the stem itself describes something ("Cephalgy.-.Weisses.Fleisch.2003", not a scene
// abbreviation like "grp-sm" under "Some.Movie.2008-GRP", nor "file_id.diz").
const tag = describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
// A document or comic is a release of its own (collections post hundreds under one subject).
if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' }
const image = IMAGE.test(input.name)
// The poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)") would make one
// release of all their uploads: a readable file name that shares no word with it says which
// release the file is (LOG.md 2026-10-09 (8)). Only when the stem itself describes something
// ("Cephalgy.-.Weisses.Fleisch.2003", not a scene abbreviation like "grp-sm" under
// "Some.Movie.2008-GRP", nor "file_id.diz"), and not for images and audio tracks, which the
// subject names as a set or album ("IMG_3086.jpg", "072 - Die Teufelsuhr.MP3").
const tag = !image && !AUDIO.test(input.name) && describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
// An image set without a counter: the image's name up to its last number, without numbers
// ("Kitchen Hottie 1 set233-76-lg" → "Kitchen Hottie set", "bm_jessica_blb_h087" → "bm jessica blb h").
const prefix = image ? stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() : ''
if (!tag && namesRelease(remainder)) {
// Bounded by the counter's total, so the uploads a poster names alike are apart while the files
// of one upload (par2 and nfo included) share it (LOG.md 2026-10-09 (9)).
const total = counterTotal(input.subject)
return {
key: hash('subject', input.poster, remainder),
key: hash('subject', input.poster, remainder, total ?? prefix),
name: remainder,
fileTotal,
source: 'subject',
@@ -178,6 +226,8 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
if (!tag && fileTotal && isPostToken(remainder)) {
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
}
// An image without words around it: its set by its name.
if (image) return { key: hash('images', input.poster, prefix), name: prefix || stem, fileTotal, source: 'stem' }
// The [i/M] total is informational, not part of the key: posters often
// upload the par2 index without a counter, and it belongs to the same post.
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }