Rebuild: image sets, documents and audio albums as releases
Picture groups made a release per image: 501k single-file releases in
the first 24 ranges of the bottom 500 groups (a.b.models 385k), ~120M
for all of them. Counters written out ('File 027 of 132', '[12of42]',
'104 of 472', '[2239-6389]', '02/17', '0097-1574'; only when the first
number is at most the second, so '[1948-12]' stays) and per-file sizes
('76.88 MB', '335555 Bytes') kept every subject unique.
- subjectRemainder removes those counters and sizes, and empty brackets.
- A subject-named release is bounded by its counter's total
(counterTotal: the first form found): uploads a poster names alike
stay apart, an upload's files (par2, nfo) together.
- Images and audio tracks don't take the poster-tag rule: the subject
names the set or album. An image set without a counter: the image's
name up to its last number, without numbers.
- Documents and comics (pdf, epub, mobi, azw3, djvu, cbr, cbz, cb7, chm,
lit, doc, docx) are a release each: collections post hundreds under
one subject ('cw-horror&mystery').
releases:regroup --group re-keys a group's indexed releases together:
cheerleaders 55,741 -> 3,165 releases, models 385,298 -> 40,313, German
audiobooks 4,106 -> 1,724, playstation 3,309 -> 2,345; comics and
e-books a release per item; boneless unchanged. Ported to edge-bench-rs;
check.ts: identical on all captures.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
59481037fb
commit
14758a5b7a
4 files changed
+272
-15
No files matched your search
+57
-7
@@ -61,14 +61,28 @@ export function fileStem(name: string): string {
|
||||
* What remains of a subject once the filename, counters, "yEnc" and sizes are
|
||||
* removed. Posters that name the release outside the quotes put it here.
|
||||
*/
|
||||
/** A file counter's text when its first number is at most its second, else the text itself (a date like [1948-12] stays). */
|
||||
const counter = (text: string, i: string, n: string) => (Number(i) <= Number(n) ? ' ' : text)
|
||||
|
||||
export function subjectRemainder(subject: string): string {
|
||||
return subject
|
||||
.replace(/"[^"]*"/g, ' ')
|
||||
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
|
||||
// Counters written out: "File 027 of 132", "[21 of 25]", "104 of 472", "[2239-6389]", "02/17",
|
||||
// "0097-1574" (each image of a set was a release of its own; LOG.md 2026-10-09 (9)).
|
||||
.replace(/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b[\])]?/gi, counter)
|
||||
.replace(/\[(\d+)-(\d+)\]/g, counter)
|
||||
.replace(/\b(\d+)\/(\d+)\b/g, counter)
|
||||
.replace(/\b(0\d*)-(\d+)\b/g, counter)
|
||||
// File sizes ("76.88 MB", "335555 Bytes") differ per file.
|
||||
.replace(/\b\d+(?:[.,]\d+)?\s*(?:[kmgt]i?b|bytes?)\b/gi, ' ')
|
||||
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
|
||||
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
|
||||
.replace(/\byEnc\b/gi, ' ')
|
||||
.replace(/[\s\-–.:|]+/g, ' ')
|
||||
// What's left of "["name.jpg" yEnc]" and of counters and sizes in brackets.
|
||||
.replace(/ ?(?:\[ ?\]|\( ?\)) ?/g, ' ')
|
||||
.replace(/ {2,}/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
@@ -107,6 +121,30 @@ function hash(...parts: (string | number | undefined)[]) {
|
||||
return digest('sha1', parts.map((p) => p ?? '').join('\n'))
|
||||
}
|
||||
|
||||
/** Image files: a set's images are named one by one ("IMG_3086.jpg"); its subject names the set. */
|
||||
const IMAGE = /\.(jpe?g|png|gif|bmp|webp|tiff?)$/i
|
||||
|
||||
/** Audio tracks: parts of an album or audiobook that the subject names ("072 - Die Teufelsuhr.MP3 - by DeckaR"). */
|
||||
const AUDIO = /\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])/i
|
||||
|
||||
/** Documents and comics: each file is an item of its own, whatever the subject says ("cw-misc [020/109] "Action Comics 710.cbr""). */
|
||||
const DOCUMENT = /\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$/i
|
||||
|
||||
/**
|
||||
* The total of the subject's file counter, in the forms subjectRemainder
|
||||
* removes, the first form found first: [i/M] or (i/M), "i of M", [i-M], i/M,
|
||||
* 0i-M (the last four only when i <= M).
|
||||
*/
|
||||
export function counterTotal(subject: string): number | undefined {
|
||||
const s = subject.replace(/"[^"]*"/g, ' ')
|
||||
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
|
||||
if (plain) return Number(plain[2] ?? plain[4])
|
||||
for (const pattern of [/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b/gi, /\[(\d+)-(\d+)\]/g, /\b(\d+)\/(\d+)\b/g, /\b(0\d*)-(\d+)\b/g]) {
|
||||
for (const m of s.matchAll(pattern)) if (Number(m[1]) <= Number(m[2])) return Number(m[2])
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** A name that says what it is: 12+ letters and digits, two words of 3+ letters. */
|
||||
function describes(stem: string): boolean {
|
||||
return stem.replace(/[^A-Za-z0-9]/g, '').length >= 12 && /[A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}/.test(stem)
|
||||
@@ -159,15 +197,25 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||||
const remainder = subjectRemainder(input.subject)
|
||||
// Unless it's the poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)"),
|
||||
// which would make one release of all their uploads: a readable file name that shares no word
|
||||
// with it says which release the file is (LOG.md 2026-10-09 (8)).
|
||||
// Only when the stem itself describes something ("Cephalgy.-.Weisses.Fleisch.2003", not a scene
|
||||
// abbreviation like "grp-sm" under "Some.Movie.2008-GRP", nor "file_id.diz").
|
||||
const tag = describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
|
||||
// A document or comic is a release of its own (collections post hundreds under one subject).
|
||||
if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' }
|
||||
const image = IMAGE.test(input.name)
|
||||
// The poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)") would make one
|
||||
// release of all their uploads: a readable file name that shares no word with it says which
|
||||
// release the file is (LOG.md 2026-10-09 (8)). Only when the stem itself describes something
|
||||
// ("Cephalgy.-.Weisses.Fleisch.2003", not a scene abbreviation like "grp-sm" under
|
||||
// "Some.Movie.2008-GRP", nor "file_id.diz"), and not for images and audio tracks, which the
|
||||
// subject names as a set or album ("IMG_3086.jpg", "072 - Die Teufelsuhr.MP3").
|
||||
const tag = !image && !AUDIO.test(input.name) && describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
|
||||
// An image set without a counter: the image's name up to its last number, without numbers
|
||||
// ("Kitchen Hottie 1 set233-76-lg" → "Kitchen Hottie set", "bm_jessica_blb_h087" → "bm jessica blb h").
|
||||
const prefix = image ? stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() : ''
|
||||
if (!tag && namesRelease(remainder)) {
|
||||
// Bounded by the counter's total, so the uploads a poster names alike are apart while the files
|
||||
// of one upload (par2 and nfo included) share it (LOG.md 2026-10-09 (9)).
|
||||
const total = counterTotal(input.subject)
|
||||
return {
|
||||
key: hash('subject', input.poster, remainder),
|
||||
key: hash('subject', input.poster, remainder, total ?? prefix),
|
||||
name: remainder,
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
@@ -178,6 +226,8 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
if (!tag && fileTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
|
||||
}
|
||||
// An image without words around it: its set by its name.
|
||||
if (image) return { key: hash('images', input.poster, prefix), name: prefix || stem, fileTotal, source: 'stem' }
|
||||
// The [i/M] total is informational, not part of the key: posters often
|
||||
// upload the par2 index without a counter, and it belongs to the same post.
|
||||
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
|
||||
|
||||
Reference in new issue
Block a user