Rebuild: image sets, documents and audio albums as releases
Picture groups made a release per image: 501k single-file releases in
the first 24 ranges of the bottom 500 groups (a.b.models 385k), ~120M
for all of them. Counters written out ('File 027 of 132', '[12of42]',
'104 of 472', '[2239-6389]', '02/17', '0097-1574'; only when the first
number is at most the second, so '[1948-12]' stays) and per-file sizes
('76.88 MB', '335555 Bytes') kept every subject unique.
- subjectRemainder removes those counters and sizes, and empty brackets.
- A subject-named release is bounded by its counter's total
(counterTotal: the first form found): uploads a poster names alike
stay apart, an upload's files (par2, nfo) together.
- Images and audio tracks don't take the poster-tag rule: the subject
names the set or album. An image set without a counter: the image's
name up to its last number, without numbers.
- Documents and comics (pdf, epub, mobi, azw3, djvu, cbr, cbz, cb7, chm,
lit, doc, docx) are a release each: collections post hundreds under
one subject ('cw-horror&mystery').
releases:regroup --group re-keys a group's indexed releases together:
cheerleaders 55,741 -> 3,165 releases, models 385,298 -> 40,313, German
audiobooks 4,106 -> 1,724, playstation 3,309 -> 2,345; comics and
e-books a release per item; boneless unchanged. Ported to edge-bench-rs;
check.ts: identical on all captures.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
59481037fb
commit
14758a5b7a
4 files changed
+272
-15
No files matched your search
@@ -21,7 +21,32 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
@flags.array({ description: 'Release ids to show in full' })
|
||||
declare show?: string[]
|
||||
|
||||
@flags.array({ description: 'Groups whose indexed releases are re-keyed together (merges show too)' })
|
||||
declare group?: string[]
|
||||
|
||||
async run() {
|
||||
for (const name of this.group ?? []) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
|
||||
[name]
|
||||
)
|
||||
const ids = rows.map((r: { release_id: string }) => r.release_id)
|
||||
const all = new Map<string, { name: string; source: string; files: number; subject?: string }>()
|
||||
for (const id of ids) {
|
||||
for (const [key, g] of await this.regroup(Number(id))) {
|
||||
const known = all.get(key)
|
||||
if (known) known.files += g.files
|
||||
else all.set(key, { ...g })
|
||||
}
|
||||
}
|
||||
const sizes = [...all.values()].map((g) => g.files)
|
||||
const single = sizes.filter((n) => n === 1).length
|
||||
this.logger.info(`${name}: ${ids.length} releases now, ${all.size} with the current rule (${single} of one file)`)
|
||||
for (const g of [...all.values()].sort((a, b) => b.files - a.files).slice(0, 6)) this.logger.info(` ${g.files} files: ${g.name.slice(0, 120)} (${g.source})`)
|
||||
const singles = [...all.values()].filter((g) => g.files === 1)
|
||||
for (let i = 0; i < Math.min(8, singles.length); i++) this.logger.info(` single: ${singles[Math.floor((i * singles.length) / 8)].subject?.slice(0, 150)}`)
|
||||
}
|
||||
if (this.group?.length) return
|
||||
for (const id of this.show ?? []) {
|
||||
const groups = await this.regroup(Number(id))
|
||||
this.logger.info(`${id}: ${groups.size} release(s) now`)
|
||||
@@ -47,13 +72,13 @@ export default class ReleasesRegroup extends BaseCommand {
|
||||
|
||||
private async regroup(id: number) {
|
||||
const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
|
||||
const groups = new Map<string, { name: string; source: string; files: number }>()
|
||||
const groups = new Map<string, { name: string; source: string; files: number; subject?: string }>()
|
||||
for (const c of chunks) {
|
||||
for (const f of decodeChunk(c.payload)) {
|
||||
const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
|
||||
const g = groups.get(r.key)
|
||||
if (g) g.files++
|
||||
else groups.set(r.key, { name: r.name, source: r.source, files: 1 })
|
||||
else groups.set(r.key, { name: r.name, source: r.source, files: 1, subject: f.subject })
|
||||
}
|
||||
}
|
||||
return groups
|
||||
|
||||
+57
-7
@@ -61,14 +61,28 @@ export function fileStem(name: string): string {
|
||||
* What remains of a subject once the filename, counters, "yEnc" and sizes are
|
||||
* removed. Posters that name the release outside the quotes put it here.
|
||||
*/
|
||||
/** A file counter's text when its first number is at most its second, else the text itself (a date like [1948-12] stays). */
|
||||
const counter = (text: string, i: string, n: string) => (Number(i) <= Number(n) ? ' ' : text)
|
||||
|
||||
export function subjectRemainder(subject: string): string {
|
||||
return subject
|
||||
.replace(/"[^"]*"/g, ' ')
|
||||
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
|
||||
// Counters written out: "File 027 of 132", "[21 of 25]", "104 of 472", "[2239-6389]", "02/17",
|
||||
// "0097-1574" (each image of a set was a release of its own; LOG.md 2026-10-09 (9)).
|
||||
.replace(/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b[\])]?/gi, counter)
|
||||
.replace(/\[(\d+)-(\d+)\]/g, counter)
|
||||
.replace(/\b(\d+)\/(\d+)\b/g, counter)
|
||||
.replace(/\b(0\d*)-(\d+)\b/g, counter)
|
||||
// File sizes ("76.88 MB", "335555 Bytes") differ per file.
|
||||
.replace(/\b\d+(?:[.,]\d+)?\s*(?:[kmgt]i?b|bytes?)\b/gi, ' ')
|
||||
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
|
||||
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
|
||||
.replace(/\byEnc\b/gi, ' ')
|
||||
.replace(/[\s\-–.:|]+/g, ' ')
|
||||
// What's left of "["name.jpg" yEnc]" and of counters and sizes in brackets.
|
||||
.replace(/ ?(?:\[ ?\]|\( ?\)) ?/g, ' ')
|
||||
.replace(/ {2,}/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
@@ -107,6 +121,30 @@ function hash(...parts: (string | number | undefined)[]) {
|
||||
return digest('sha1', parts.map((p) => p ?? '').join('\n'))
|
||||
}
|
||||
|
||||
/** Image files: a set's images are named one by one ("IMG_3086.jpg"); its subject names the set. */
|
||||
const IMAGE = /\.(jpe?g|png|gif|bmp|webp|tiff?)$/i
|
||||
|
||||
/** Audio tracks: parts of an album or audiobook that the subject names ("072 - Die Teufelsuhr.MP3 - by DeckaR"). */
|
||||
const AUDIO = /\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])/i
|
||||
|
||||
/** Documents and comics: each file is an item of its own, whatever the subject says ("cw-misc [020/109] "Action Comics 710.cbr""). */
|
||||
const DOCUMENT = /\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$/i
|
||||
|
||||
/**
|
||||
* The total of the subject's file counter, in the forms subjectRemainder
|
||||
* removes, the first form found first: [i/M] or (i/M), "i of M", [i-M], i/M,
|
||||
* 0i-M (the last four only when i <= M).
|
||||
*/
|
||||
export function counterTotal(subject: string): number | undefined {
|
||||
const s = subject.replace(/"[^"]*"/g, ' ')
|
||||
const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
|
||||
if (plain) return Number(plain[2] ?? plain[4])
|
||||
for (const pattern of [/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b/gi, /\[(\d+)-(\d+)\]/g, /\b(\d+)\/(\d+)\b/g, /\b(0\d*)-(\d+)\b/g]) {
|
||||
for (const m of s.matchAll(pattern)) if (Number(m[1]) <= Number(m[2])) return Number(m[2])
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** A name that says what it is: 12+ letters and digits, two words of 3+ letters. */
|
||||
function describes(stem: string): boolean {
|
||||
return stem.replace(/[^A-Za-z0-9]/g, '').length >= 12 && /[A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}/.test(stem)
|
||||
@@ -159,15 +197,25 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||||
const remainder = subjectRemainder(input.subject)
|
||||
// Unless it's the poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)"),
|
||||
// which would make one release of all their uploads: a readable file name that shares no word
|
||||
// with it says which release the file is (LOG.md 2026-10-09 (8)).
|
||||
// Only when the stem itself describes something ("Cephalgy.-.Weisses.Fleisch.2003", not a scene
|
||||
// abbreviation like "grp-sm" under "Some.Movie.2008-GRP", nor "file_id.diz").
|
||||
const tag = describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
|
||||
// A document or comic is a release of its own (collections post hundreds under one subject).
|
||||
if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' }
|
||||
const image = IMAGE.test(input.name)
|
||||
// The poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)") would make one
|
||||
// release of all their uploads: a readable file name that shares no word with it says which
|
||||
// release the file is (LOG.md 2026-10-09 (8)). Only when the stem itself describes something
|
||||
// ("Cephalgy.-.Weisses.Fleisch.2003", not a scene abbreviation like "grp-sm" under
|
||||
// "Some.Movie.2008-GRP", nor "file_id.diz"), and not for images and audio tracks, which the
|
||||
// subject names as a set or album ("IMG_3086.jpg", "072 - Die Teufelsuhr.MP3").
|
||||
const tag = !image && !AUDIO.test(input.name) && describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
|
||||
// An image set without a counter: the image's name up to its last number, without numbers
|
||||
// ("Kitchen Hottie 1 set233-76-lg" → "Kitchen Hottie set", "bm_jessica_blb_h087" → "bm jessica blb h").
|
||||
const prefix = image ? stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() : ''
|
||||
if (!tag && namesRelease(remainder)) {
|
||||
// Bounded by the counter's total, so the uploads a poster names alike are apart while the files
|
||||
// of one upload (par2 and nfo included) share it (LOG.md 2026-10-09 (9)).
|
||||
const total = counterTotal(input.subject)
|
||||
return {
|
||||
key: hash('subject', input.poster, remainder),
|
||||
key: hash('subject', input.poster, remainder, total ?? prefix),
|
||||
name: remainder,
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
@@ -178,6 +226,8 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
if (!tag && fileTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
|
||||
}
|
||||
// An image without words around it: its set by its name.
|
||||
if (image) return { key: hash('images', input.poster, prefix), name: prefix || stem, fileTotal, source: 'stem' }
|
||||
// The [i/M] total is informational, not part of the key: posters often
|
||||
// upload the par2 index without a counter, and it belongs to the same post.
|
||||
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
|
||||
|
||||
@@ -196,4 +196,38 @@ test.group('release / par2 names', () => {
|
||||
assert.equal(scene.name, 'Some Movie 2008 DVDRip XviD GRP')
|
||||
assert.equal(id('Better.Jpeg.v2.0-DOA [02/10] - "file_id.diz" yEnc', 'file_id.diz').key, id('Better.Jpeg.v2.0-DOA [03/10] - "doa-bj20.zip" yEnc', 'doa-bj20.zip').key)
|
||||
})
|
||||
test('image sets: counters written out, sizes and the set by its counter or name', ({ assert }) => {
|
||||
const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })!
|
||||
// "File 027 of 132": one set, not a release per image; another set of the same title apart by its total.
|
||||
const a = id('What about this? - File 027 of 132: "DSC_027.jpg" yEnc', 'DSC_027.jpg')
|
||||
const b = id('What about this? - File 028 of 132: "DSC_028.jpg" yEnc', 'DSC_028.jpg')
|
||||
const other = id('What about this? - File 001 of 90: "DSC_001.jpg" yEnc', 'DSC_001.jpg')
|
||||
assert.equal(a.key, b.key)
|
||||
assert.notEqual(a.key, other.key)
|
||||
assert.equal(a.name, 'What about this?')
|
||||
// No counter: the image's name up to its last number.
|
||||
const k1 = id('Proud to be a poster "Kitchen Hottie 1 set233-76-lg.jpg" yEnc', 'Kitchen Hottie 1 set233-76-lg.jpg')
|
||||
const k2 = id('Proud to be a poster "Kitchen Hottie 1 set233-77-lg.jpg" yEnc', 'Kitchen Hottie 1 set233-77-lg.jpg')
|
||||
const k3 = id('Proud to be a poster "Office Chair set12-01-lg.jpg" yEnc', 'Office Chair set12-01-lg.jpg')
|
||||
assert.equal(k1.key, k2.key)
|
||||
assert.notEqual(k1.key, k3.key)
|
||||
// Per-file sizes and other counter forms don't split a post.
|
||||
assert.equal(subjectRemainder('Minutemen (001/104) "x.cbr" - 76.88 MB - yEnc'), 'Minutemen')
|
||||
assert.equal(subjectRemainder('Babes P A R T 3 ("Charlotte 03 16.jpg" 0097-1574 156.87 kB-273.03 MB) yEnc'), 'Babes P A R T 3')
|
||||
assert.equal(subjectRemainder('*MaryKate* - OfficeChair [23of61] - "OfficeChair023.jpg" yEnc 187924 bytes'), '*MaryKate* OfficeChair')
|
||||
assert.equal(subjectRemainder('Adventure Comics [1948-12] - "x.rar"'), 'Adventure Comics [1948 12]')
|
||||
})
|
||||
|
||||
test('documents are a release each; audio tracks stay with their album', ({ assert }) => {
|
||||
const id = (subject: string, name: string) => releaseIdentity({ poster: 'p', subject, name, weak: false })!
|
||||
const c1 = id('cw-horror&mystery [10/28] "Vampirella Magazine 061.cbr" yEnc', 'Vampirella Magazine 061.cbr')
|
||||
const c2 = id('cw-horror&mystery [11/28] "Mystery Tales 12.cbr" yEnc', 'Mystery Tales 12.cbr')
|
||||
assert.notEqual(c1.key, c2.key)
|
||||
assert.equal(c2.name, 'Mystery Tales 12')
|
||||
const t1 = id('(Geisterjäger John Sinclair Hörspiele Vol. 2) [04/59] - "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR" yEnc', '042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR')
|
||||
const t2 = id('(Geisterjäger John Sinclair Hörspiele Vol. 2) [34/59] - "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR" yEnc', '072 - 027 - Die Teufelsuhr.MP3 - by DeckaR')
|
||||
const par2 = id('(Geisterjäger John Sinclair Hörspiele Vol. 2) [59/59] - "Hörspiele Vol. 2.par2" yEnc', 'Hörspiele Vol. 2.par2')
|
||||
assert.equal(t1.key, t2.key)
|
||||
assert.equal(t1.key, par2.key)
|
||||
})
|
||||
})
|
||||
@@ -76,10 +76,11 @@ static SEPARATORS: LazyLock<Regex> = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80
|
||||
pub fn subject_remainder_re(subject: &str) -> String {
|
||||
let s = replace("ED, subject, " ", true);
|
||||
let s = replace(&COUNTERS, &s, " ", true);
|
||||
let s = remove_counters_and_sizes(&s);
|
||||
let s = replace(&YENC_END, &s, " ", false);
|
||||
let s = replace(&YENC, &s, " ", true);
|
||||
let s = replace(&SEPARATORS, &s, " ", true);
|
||||
js::trim(&s).to_string()
|
||||
finish_remainder(&s)
|
||||
}
|
||||
|
||||
fn eq_ci(a: &[u8], b: &[u8]) -> bool {
|
||||
@@ -262,7 +263,7 @@ pub fn subject_remainder(subject: &str) -> String {
|
||||
i += 1;
|
||||
}
|
||||
// Only ASCII was replaced, between whole characters.
|
||||
let mut s3 = String::from_utf8(s2).unwrap();
|
||||
let mut s3 = remove_counters_and_sizes(&String::from_utf8(s2).unwrap());
|
||||
// \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits)
|
||||
let b3 = s3.as_bytes();
|
||||
if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) {
|
||||
@@ -296,7 +297,107 @@ pub fn subject_remainder(subject: &str) -> String {
|
||||
}
|
||||
i += c.len_utf8();
|
||||
}
|
||||
js::trim(&s4).to_string()
|
||||
finish_remainder(&s4)
|
||||
}
|
||||
|
||||
static OF_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b[\])]?"));
|
||||
static DASH_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)-([0-9]+)\]"));
|
||||
static SLASH_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\b([0-9]+)/([0-9]+)\b"));
|
||||
static ZERO_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\b(0[0-9]*)-([0-9]+)\b"));
|
||||
static SIZES: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b[0-9]+(?:[.,][0-9]+)?SP*(?:[kmgt]i?b|bytes?)\b"));
|
||||
static EMPTY_BRACKETS: LazyLock<Regex> = LazyLock::new(|| re(r" ?(?:\[ ?\]|\( ?\)) ?"));
|
||||
|
||||
fn number(digits: &[u8]) -> f64 {
|
||||
std::str::from_utf8(digits).unwrap().parse::<f64>().unwrap()
|
||||
}
|
||||
|
||||
/// `s.replace(/…/g, counter)`: the match goes when its first number is at most its second.
|
||||
fn replace_counters(re: &Regex, s: &str) -> String {
|
||||
if !re.is_match(s.as_bytes()) {
|
||||
return s.to_string();
|
||||
}
|
||||
let out = re.replace_all(s.as_bytes(), |c: ®ex::bytes::Captures| if number(&c[1]) <= number(&c[2]) { b" ".to_vec() } else { c[0].to_vec() });
|
||||
String::from_utf8(out.into_owned()).unwrap()
|
||||
}
|
||||
|
||||
/// The written-out counters, file sizes and empty brackets subjectRemainder removes after the [i/M] ones.
|
||||
fn remove_counters_and_sizes(s: &str) -> String {
|
||||
let s = replace_counters(&OF_COUNTER, s);
|
||||
let s = replace_counters(&DASH_COUNTER, &s);
|
||||
let s = replace_counters(&SLASH_COUNTER, &s);
|
||||
let s = replace_counters(&ZERO_COUNTER, &s);
|
||||
replace(&SIZES, &s, " ", true)
|
||||
}
|
||||
|
||||
/// The end of subjectRemainder: what's left of brackets, double spaces, trim (after the separators became single spaces).
|
||||
fn finish_remainder(s: &str) -> String {
|
||||
let s = if s.contains(['[', '(']) { replace(&EMPTY_BRACKETS, s, " ", true) } else { s.to_string() };
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
if c == ' ' && out.ends_with(' ') {
|
||||
continue;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
js::trim(&out).to_string()
|
||||
}
|
||||
|
||||
/// counterTotal: the subject's file counter total, first form found first.
|
||||
pub fn counter_total(subject: &str) -> Option<String> {
|
||||
let s = replace("ED, subject, " ", true);
|
||||
if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) {
|
||||
let total = c.get(2).or(c.get(4)).unwrap().as_bytes();
|
||||
return Some(js_number_string(number(total)));
|
||||
}
|
||||
for re in [&*OF_COUNTER_OPEN, &*DASH_COUNTER, &*SLASH_COUNTER, &*ZERO_COUNTER] {
|
||||
for c in re.captures_iter(s.as_bytes()) {
|
||||
if number(&c[1]) <= number(&c[2]) {
|
||||
return Some(js_number_string(number(&c[2])));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
static COUNTERS_CAPTURE: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]|\(([0-9]+)/([0-9]+)\)"));
|
||||
/// The "of" counter without its closing bracket (as counterTotal's pattern).
|
||||
static OF_COUNTER_OPEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b"));
|
||||
|
||||
/// `String(n)` of a counter's number (integers as JS prints them).
|
||||
fn js_number_string(n: f64) -> String {
|
||||
if n.fract() == 0.0 && n.abs() < 1e21 {
|
||||
format!("{}", n as i128)
|
||||
} else {
|
||||
format!("{n}")
|
||||
}
|
||||
}
|
||||
|
||||
static IMAGE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(jpe?g|png|gif|bmp|webp|tiff?)$"));
|
||||
static AUDIO: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])"));
|
||||
static DOCUMENT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$"));
|
||||
|
||||
/// An image set's name without a counter: up to the last number, without numbers.
|
||||
fn image_prefix(stem: &str) -> String {
|
||||
let b = stem.as_bytes();
|
||||
let cut = match b.iter().rposition(u8::is_ascii_digit) {
|
||||
Some(last) => b[..=last].iter().rposition(|c| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0),
|
||||
None => b.len(),
|
||||
};
|
||||
let head = &stem[..cut];
|
||||
let mut out = String::with_capacity(head.len());
|
||||
let mut sep = false;
|
||||
for c in head.chars() {
|
||||
if c.is_ascii_digit() || js::is_space(c) || matches!(c, '.' | '_' | '-') {
|
||||
sep = true;
|
||||
} else {
|
||||
if sep && !out.is_empty() {
|
||||
out.push(' ');
|
||||
}
|
||||
out.push(c);
|
||||
sep = false;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn is_random_token(token: &str) -> bool {
|
||||
@@ -412,16 +513,29 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||
let file_total = file_counter_total(subject);
|
||||
let stem = file_stem(name);
|
||||
let remainder = subject_remainder(subject);
|
||||
// The poster's constant tag outside the quotes: the file's stem says which upload it is.
|
||||
let tag = describes(&stem) && !looks_random(&stem) && is_poster_tag(&remainder, &stem);
|
||||
// A document or comic is a release of its own.
|
||||
if test(&DOCUMENT, name) {
|
||||
let shown = replace(&DOCUMENT, &stem, "", false);
|
||||
let name = if shown.is_empty() { stem.clone() } else { shown };
|
||||
return Release { key: hash(&["stem", poster, &stem]), name, source: "stem", file_total };
|
||||
}
|
||||
let image = test(&IMAGE, name);
|
||||
// The poster's constant tag outside the quotes: the file's stem says which upload it is (not for images and audio).
|
||||
let tag = !image && !test(&AUDIO, name) && describes(&stem) && !looks_random(&stem) && is_poster_tag(&remainder, &stem);
|
||||
let prefix = if image { image_prefix(&stem) } else { String::new() };
|
||||
if !tag && names_release(&remainder) {
|
||||
return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total };
|
||||
let bound = counter_total(subject).unwrap_or_else(|| prefix.clone());
|
||||
return Release { key: hash(&["subject", poster, &remainder, &bound]), name: remainder, source: "subject", file_total };
|
||||
}
|
||||
if let Some(total) = file_total.filter(|&n| n != 0 && !tag) {
|
||||
if is_post_token(&remainder) {
|
||||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||||
}
|
||||
}
|
||||
if image {
|
||||
let name = if prefix.is_empty() { stem.clone() } else { prefix.clone() };
|
||||
return Release { key: hash(&["images", poster, &prefix]), name, source: "stem", file_total };
|
||||
}
|
||||
Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total }
|
||||
}
|
||||
|
||||
@@ -801,6 +915,40 @@ mod tests {
|
||||
assert!(!is_poster_tag("Audiofiles DIE ZEIT 35", "35_China_dl"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sets_documents_and_audio_as_in_the_typescript() {
|
||||
let id = |subject: &str, name: &str| release_identity("p", subject, name);
|
||||
let a = id(r#"What about this? - File 027 of 132: "DSC_027.jpg" yEnc"#, "DSC_027.jpg");
|
||||
let b = id(r#"What about this? - File 028 of 132: "DSC_028.jpg" yEnc"#, "DSC_028.jpg");
|
||||
let other = id(r#"What about this? - File 001 of 90: "DSC_001.jpg" yEnc"#, "DSC_001.jpg");
|
||||
assert_eq!(a.key, b.key);
|
||||
assert_ne!(a.key, other.key);
|
||||
assert_eq!(a.name, "What about this?");
|
||||
let k1 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-76-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-76-lg.jpg");
|
||||
let k2 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-77-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-77-lg.jpg");
|
||||
let k3 = id(r#"Proud to be a poster "Office Chair set12-01-lg.jpg" yEnc"#, "Office Chair set12-01-lg.jpg");
|
||||
assert_eq!(k1.key, k2.key);
|
||||
assert_ne!(k1.key, k3.key);
|
||||
for (subject, expected) in [
|
||||
(r#"Minutemen (001/104) "x.cbr" - 76.88 MB - yEnc"#, "Minutemen"),
|
||||
(r#"Babes P A R T 3 ("Charlotte 03 16.jpg" 0097-1574 156.87 kB-273.03 MB) yEnc"#, "Babes P A R T 3"),
|
||||
(r#"*MaryKate* - OfficeChair [23of61] - "OfficeChair023.jpg" yEnc 187924 bytes"#, "*MaryKate* OfficeChair"),
|
||||
(r#"Adventure Comics [1948-12] - "x.rar""#, "Adventure Comics [1948 12]"),
|
||||
] {
|
||||
assert_eq!(subject_remainder(subject), expected, "{subject}");
|
||||
assert_eq!(subject_remainder_re(subject), expected, "{subject}");
|
||||
}
|
||||
let c1 = id(r#"cw-horror&mystery [10/28] "Vampirella Magazine 061.cbr" yEnc"#, "Vampirella Magazine 061.cbr");
|
||||
let c2 = id(r#"cw-horror&mystery [11/28] "Mystery Tales 12.cbr" yEnc"#, "Mystery Tales 12.cbr");
|
||||
assert_ne!(c1.key, c2.key);
|
||||
assert_eq!(c2.name, "Mystery Tales 12");
|
||||
let t1 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [04/59] - "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR" yEnc"#, "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR");
|
||||
let t2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [34/59] - "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR" yEnc"#, "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR");
|
||||
let par2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [59/59] - "Hörspiele Vol. 2.par2" yEnc"#, "Hörspiele Vol. 2.par2");
|
||||
assert_eq!(t1.key, t2.key);
|
||||
assert_eq!(t1.key, par2.key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_names() {
|
||||
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||||
|
||||
Reference in new issue
Block a user