Daan MeijerandClaude Opus 5.5
5109481d67
User: Rust is the product (option a). tools/edge-bench-rs moves to edge/
(crate edge; binaries edge-node, the worker, and edge-bench, the replay).
CI's new job edge runs cargo test and edge/check-fixtures.sh: both
RangeIndexers on three committed captures (edge/fixtures, 25k overview
lines each of a.b.models, a.b.ath and a.b.boneless in 5 chunks, 3.5 MB),
compared field by field with check.ts; deploy waits for it. The
TypeScript RangeIndexer stays as check.ts's reference. README: what it
is now; the benchmark findings as history.
Random names: looksRandom missed long tokens of capitals and digits
("51294P1CUKR2R35X", release 74539068, readable and a Newznab title).
12+ capitals and digits switching 6+ times between them are random
(COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice); both
implementations, identical on the fixtures.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
245 lines
15 KiB
TypeScript
245 lines
15 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { fileStem, releaseIdentity, subjectRemainder } from '#lib/release/index'
|
|
import { looksLikeFilename, weakFileKey } from '#lib/headers/index'
|
|
|
|
test.group('release / fileStem', () => {
|
|
test('reduces all files of a set to the same stem', ({ assert }) => {
|
|
const names = [
|
|
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst',
|
|
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.par2',
|
|
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.vol03+04.par2',
|
|
]
|
|
assert.deepEqual(new Set(names.map(fileStem)), new Set(['1790546808_nicovideo_jp_watch_sm26679715']))
|
|
})
|
|
|
|
test('handles rar, split and par2 volume naming', ({ assert }) => {
|
|
assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0')
|
|
assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024')
|
|
assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024')
|
|
assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265')
|
|
assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264')
|
|
assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024')
|
|
assert.equal(fileStem('Some.Movie.2024.nfo'), 'Some.Movie.2024')
|
|
assert.equal(fileStem('00-artist-album-web-2026.m3u'), '00-artist-album-web-2026')
|
|
assert.equal(fileStem('f6dd6b156692a52b.bin'), 'f6dd6b156692a52b')
|
|
})
|
|
|
|
test('never reduces a name to nothing', ({ assert }) => {
|
|
assert.equal(fileStem('.par2'), '.par2')
|
|
})
|
|
})
|
|
|
|
test.group('release / subjectRemainder', () => {
|
|
test('is empty when the subject only has counters, filename and size', ({ assert }) => {
|
|
assert.equal(subjectRemainder('[1/4] - "x.tar.zst" yEnc 15411409'), '')
|
|
assert.equal(subjectRemainder('. - [00/88] - "60SHkTQSvn5RZGExwju2.nzb" yEnc'), '')
|
|
})
|
|
|
|
test('keeps a release name written outside the quotes', ({ assert }) => {
|
|
assert.equal(
|
|
subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'),
|
|
'Some Show S01E02 1080p'
|
|
)
|
|
})
|
|
})
|
|
|
|
test.group('release / releaseIdentity', () => {
|
|
test('groups a par2 set with its archive by stem, per poster', ({ assert }) => {
|
|
const base = { poster: 'p', weak: false }
|
|
const a = releaseIdentity({ ...base, subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
|
|
const b = releaseIdentity({ ...base, subject: '[3/4] - "x.tar.zst.vol01+02.par2" yEnc 2', name: 'x.tar.zst.vol01+02.par2' })!
|
|
assert.equal(a.key, b.key)
|
|
assert.equal(a.name, 'x')
|
|
assert.equal(a.fileTotal, 4)
|
|
const other = releaseIdentity({ ...base, poster: 'q', subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
|
|
assert.notEqual(a.key, other.key)
|
|
})
|
|
|
|
test('a par2 index posted without [i/M] counter joins its release', ({ assert }) => {
|
|
const base = { poster: 'nicovideo_archiver', weak: false }
|
|
const archive = releaseIdentity({ ...base, subject: '[1/6] - "x.tar.zst" yEnc 105054576', name: 'x.tar.zst' })!
|
|
const index = releaseIdentity({ ...base, subject: '"x.tar.zst.par2" yEnc 3100', name: 'x.tar.zst.par2' })!
|
|
assert.equal(index.key, archive.key)
|
|
assert.equal(archive.fileTotal, 6)
|
|
assert.isUndefined(index.fileTotal)
|
|
})
|
|
|
|
test('uses the subject text when it names the release', ({ assert }) => {
|
|
const base = { poster: 'p', weak: false }
|
|
const a = releaseIdentity({ ...base, subject: 'My.Release [01/40] - "abc.part01.rar" yEnc', name: 'abc.part01.rar' })!
|
|
const b = releaseIdentity({ ...base, subject: 'My.Release [40/40] - "abc.par2" yEnc', name: 'abc.par2' })!
|
|
assert.equal(a.key, b.key)
|
|
assert.equal(a.name, 'My Release')
|
|
})
|
|
|
|
test('obfuscated volumes group by stem regardless of the random poster', ({ assert }) => {
|
|
const a = releaseIdentity({ poster: '3Uzx7TTSFpLzz@ngPost.com', subject: '"lJzu.part033.rar" yEnc', name: 'lJzu.part033.rar', weak: true })!
|
|
const b = releaseIdentity({ poster: 'bGLd1oaXRdrK1@ngPost.com', subject: '"lJzu.part034.rar" yEnc', name: 'lJzu.part034.rar', weak: true })!
|
|
assert.equal(a.key, b.key)
|
|
})
|
|
|
|
test('capitals and digits switching all over a token are random', async ({ assert }) => {
|
|
const { looksRandom } = await import('#lib/release/index')
|
|
assert.isTrue(looksRandom('51294P1CUKR2R35X'))
|
|
assert.isTrue(looksRandom('51294P1CUKR2R35X.part01.rar'.replace(/\.part01\.rar$/, '')))
|
|
assert.isFalse(looksRandom('COD4MW2REMASTERED'))
|
|
assert.isFalse(looksRandom('BEATLES1964REMASTER'))
|
|
assert.isFalse(looksRandom('HDTV1080P'))
|
|
})
|
|
|
|
test('random extensionless names cannot be grouped by name', ({ assert }) => {
|
|
const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'
|
|
assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true }))
|
|
})
|
|
})
|
|
|
|
test.group('subject / weakFileKey', () => {
|
|
test('ignores random per-article names, keeps real filenames', ({ assert }) => {
|
|
assert.isFalse(looksLikeFilename('204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'))
|
|
assert.isTrue(looksLikeFilename('lJzuoIKA3T8xjzAD3vqtw0.part033.rar'))
|
|
// Same file, different random names per article: same key.
|
|
assert.equal(weakFileKey('aaaa1111', 5000, 7), weakFileKey('bbbb2222', 5000, 7))
|
|
// Equal-sized volumes with real names: different keys.
|
|
assert.notEqual(weakFileKey('x.part01.rar', 5000, 7), weakFileKey('x.part02.rar', 5000, 7))
|
|
assert.notEqual(weakFileKey('aaaa1111', 5000, 7), weakFileKey('aaaa1111', 5001, 7))
|
|
})
|
|
})
|
|
|
|
test.group('release / rarReleaseIdentity', () => {
|
|
test('groups volumes by the shared top-level directory', async ({ assert }) => {
|
|
const { rarReleaseIdentity } = await import('#lib/release/index')
|
|
const first = rarReleaseIdentity([
|
|
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/00-cover.jpg', size: 589382 },
|
|
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/01-track.flac', size: 30_000_000 },
|
|
])!
|
|
const later = rarReleaseIdentity([{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/03-track.flac', size: 29_000_000 }])!
|
|
assert.equal(first.key, later.key)
|
|
assert.equal(first.name, 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM')
|
|
assert.equal(first.source, 'rar')
|
|
})
|
|
|
|
test('without a directory, volumes of one file share its first entry', async ({ assert }) => {
|
|
const { rarReleaseIdentity } = await import('#lib/release/index')
|
|
const a = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
|
|
const b = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
|
|
assert.equal(a.key, b.key)
|
|
assert.equal(a.name, 'Some.Show.S01E01.1080p')
|
|
assert.isNull(rarReleaseIdentity([]))
|
|
})
|
|
})
|
|
|
|
test.group('release / namesRelease', () => {
|
|
test('random per-file tags and poster ids are not a release name', async ({ assert }) => {
|
|
const { namesRelease, subjectRemainder, releaseIdentity } = await import('#lib/release/index')
|
|
const subject = 'elys-de81bcabef5faf6a - [Nyh5Q07Y] "JkpNIFg7En50zU1U" yEnc'
|
|
assert.isFalse(namesRelease(subjectRemainder(subject)))
|
|
const a = releaseIdentity({ poster: 'elys', subject, name: 'JkpNIFg7En50zU1U', weak: false })!
|
|
assert.equal(a.source, 'stem')
|
|
assert.isTrue(namesRelease(subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc')))
|
|
assert.isTrue(namesRelease('My Release'))
|
|
})
|
|
})
|
|
|
|
test.group('release / post id', () => {
|
|
test('groups random-named files of one post by poster, post id and file count', async ({ assert }) => {
|
|
const { releaseIdentity, isPostToken } = await import('#lib/release/index')
|
|
const base = { poster: 'jW344UtC8R2bZEegrVLz@x', weak: false }
|
|
const a = releaseIdentity({ ...base, subject: '83110629-n [10/18] - "eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k" yEnc', name: 'eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k' })!
|
|
const b = releaseIdentity({ ...base, subject: '83110629-n [01/18] - "2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0" yEnc', name: '2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0' })!
|
|
assert.equal(a.key, b.key)
|
|
assert.equal(a.source, 'post')
|
|
assert.equal(a.fileTotal, 18)
|
|
const other = releaseIdentity({ ...base, subject: '83110630-n [01/18] - "x" yEnc', name: 'x' })!
|
|
assert.notEqual(a.key, other.key)
|
|
|
|
assert.isTrue(isPostToken('83110629 n'))
|
|
assert.isFalse(isPostToken('PRiVATE'))
|
|
assert.isFalse(isPostToken(''))
|
|
// Without a counter there is nothing to bound the post: fall back to the stem.
|
|
assert.equal(releaseIdentity({ ...base, subject: '83110629-n - "x.bin" yEnc', name: 'x.bin' })!.source, 'stem')
|
|
})
|
|
})
|
|
|
|
test.group('release / par2 names', () => {
|
|
test('names a season pack after what its episodes share', async ({ assert }) => {
|
|
const { par2ReleaseName } = await import('#lib/release/index')
|
|
const eps = Array.from({ length: 20 }, (_, i) =>
|
|
`SEAL.Team.S03E${String(i + 1).padStart(2, '0')}.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato.mkv`)
|
|
assert.equal(par2ReleaseName(eps), 'SEAL.Team.S03.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato')
|
|
// Movie plus sample and nfo: the movie's (longest) name.
|
|
assert.equal(
|
|
par2ReleaseName(['Some.Movie.2024.1080p.BluRay.x264-GRP.mkv', 'Some.Movie.2024.1080p.BluRay.x264-GRP.nfo', 'sample.mkv']),
|
|
'Some.Movie.2024.1080p.BluRay.x264-GRP'
|
|
)
|
|
// Numbered parts: the shared name without the part number.
|
|
assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP')
|
|
assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024')
|
|
})
|
|
test("a poster's constant tag doesn't make one release of all their uploads", ({ assert }) => {
|
|
const id = (poster: string, subject: string, name: string) => releaseIdentity({ poster, subject, name, weak: false })!
|
|
// [KiLL…] tag on every upload: each album by its stem, its nfo included.
|
|
const kill = 'www.ggb.6x.to <darkness@earth.de>'
|
|
const a = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc', 'Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00')
|
|
const aNfo = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc', 'Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo')
|
|
const b = id(kill, '[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc', 'Shnarph.-.Lautlos.2008[[g.6x.to]].r00')
|
|
assert.equal(a.key, aNfo.key)
|
|
assert.notEqual(a.key, b.key)
|
|
assert.equal(a.source, 'stem')
|
|
// A tag taken for a post id: "(PowBeertje80 Posted)" with the same [i/M] on every album.
|
|
const pow = (album: string) => id('pow', `(PowBeertje80 Posted) [01/026] - "${album}.part01.rar" yEnc`, `${album}.part01.rar`)
|
|
assert.notEqual(pow('Bravo Hits Vol.14 2CD -1996-').key, pow('Bravo Hits Vol.33 2CD -2001-').key)
|
|
})
|
|
|
|
test('the text outside the quotes still groups when it describes the files, or is a per-post id', ({ assert }) => {
|
|
const id = (subject: string, name: string) => releaseIdentity({ poster: 'p', subject, name, weak: false })!
|
|
// A word in common ("ketchup"): one album of many track files.
|
|
const t1 = id('!!!!! FTA www.ftaclub.net BodyCrew Presents VA-Ketchup_Dance2008 !!!!! [01/28] - "fta-01-a_track-2008.mp3" yEnc', 'fta-01-a_track-2008.mp3')
|
|
const t2 = id('!!!!! FTA www.ftaclub.net BodyCrew Presents VA-Ketchup_Dance2008 !!!!! [02/28] - "fta-02-coyote-ketchup_mix.mp3" yEnc', 'fta-02-coyote-ketchup_mix.mp3')
|
|
assert.equal(t1.key, t2.key)
|
|
assert.equal(t1.source, 'subject')
|
|
// A number in common ("35"), and a per-post id ("FTD769997").
|
|
assert.equal(id('[Audiofiles_DIE_ZEIT_35] [01/24] - "35_China_dl.mp3" yEnc', '35_China_dl.mp3').key, id('[Audiofiles_DIE_ZEIT_35] [02/24] - "35_Bamberg_dl.mp3" yEnc', '35_Bamberg_dl.mp3').key)
|
|
assert.equal(id('FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc', 'Prison.Break.S04E02.720p.part01.rar').key, id('FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc', 'Prison.Break.S04E02.720p.part.par2').key)
|
|
// A scene name with abbreviated file names, and a generically named sidecar.
|
|
const scene = id('Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc', 'grp-sm.part01.rar')
|
|
assert.equal(scene.name, 'Some Movie 2008 DVDRip XviD GRP')
|
|
assert.equal(id('Better.Jpeg.v2.0-DOA [02/10] - "file_id.diz" yEnc', 'file_id.diz').key, id('Better.Jpeg.v2.0-DOA [03/10] - "doa-bj20.zip" yEnc', 'doa-bj20.zip').key)
|
|
})
|
|
test('image sets: counters written out, sizes and the set by its counter or name', ({ assert }) => {
|
|
const id = (subject: string, name: string, poster = 'p') => releaseIdentity({ poster, subject, name, weak: false })!
|
|
// "File 027 of 132": one set, not a release per image; another set of the same title apart by its total.
|
|
const a = id('What about this? - File 027 of 132: "DSC_027.jpg" yEnc', 'DSC_027.jpg')
|
|
const b = id('What about this? - File 028 of 132: "DSC_028.jpg" yEnc', 'DSC_028.jpg')
|
|
const other = id('What about this? - File 001 of 90: "DSC_001.jpg" yEnc', 'DSC_001.jpg')
|
|
assert.equal(a.key, b.key)
|
|
assert.notEqual(a.key, other.key)
|
|
assert.equal(a.name, 'What about this?')
|
|
// No counter: the image's name up to its last number.
|
|
const k1 = id('Proud to be a poster "Kitchen Hottie 1 set233-76-lg.jpg" yEnc', 'Kitchen Hottie 1 set233-76-lg.jpg')
|
|
const k2 = id('Proud to be a poster "Kitchen Hottie 1 set233-77-lg.jpg" yEnc', 'Kitchen Hottie 1 set233-77-lg.jpg')
|
|
const k3 = id('Proud to be a poster "Office Chair set12-01-lg.jpg" yEnc', 'Office Chair set12-01-lg.jpg')
|
|
assert.equal(k1.key, k2.key)
|
|
assert.notEqual(k1.key, k3.key)
|
|
// Per-file sizes and other counter forms don't split a post.
|
|
assert.equal(subjectRemainder('Minutemen (001/104) "x.cbr" - 76.88 MB - yEnc'), 'Minutemen')
|
|
assert.equal(subjectRemainder('Babes P A R T 3 ("Charlotte 03 16.jpg" 0097-1574 156.87 kB-273.03 MB) yEnc'), 'Babes P A R T 3')
|
|
assert.equal(subjectRemainder('*MaryKate* - OfficeChair [23of61] - "OfficeChair023.jpg" yEnc 187924 bytes'), '*MaryKate* OfficeChair')
|
|
assert.equal(subjectRemainder('Adventure Comics [1948-12] - "x.rar"'), 'Adventure Comics [1948 12]')
|
|
})
|
|
|
|
test('documents are a release each; audio tracks stay with their album', ({ assert }) => {
|
|
const id = (subject: string, name: string) => releaseIdentity({ poster: 'p', subject, name, weak: false })!
|
|
const c1 = id('cw-horror&mystery [10/28] "Vampirella Magazine 061.cbr" yEnc', 'Vampirella Magazine 061.cbr')
|
|
const c2 = id('cw-horror&mystery [11/28] "Mystery Tales 12.cbr" yEnc', 'Mystery Tales 12.cbr')
|
|
assert.notEqual(c1.key, c2.key)
|
|
assert.equal(c2.name, 'Mystery Tales 12')
|
|
const t1 = id('(Geisterjäger John Sinclair Hörspiele Vol. 2) [04/59] - "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR" yEnc', '042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR')
|
|
const t2 = id('(Geisterjäger John Sinclair Hörspiele Vol. 2) [34/59] - "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR" yEnc', '072 - 027 - Die Teufelsuhr.MP3 - by DeckaR')
|
|
const par2 = id('(Geisterjäger John Sinclair Hörspiele Vol. 2) [59/59] - "Hörspiele Vol. 2.par2" yEnc', 'Hörspiele Vol. 2.par2')
|
|
assert.equal(t1.key, t2.key)
|
|
assert.equal(t1.key, par2.key)
|
|
// A cover without words around it joins its stem's archive.
|
|
assert.equal(id('[2/6] - "x.jpg" yEnc', 'x.jpg').key, id('[1/6] - "x.part01.rar" yEnc', 'x.part01.rar').key)
|
|
})
|
|
})
|