Daan MeijerandClaude Opus 5.5
e00e4f3988
The files of one upload often became several releases whose names
differ slightly: scene tracks apart from their album's nfo/sfv/par2,
"Vermaechtnis" beside a garbled "Vermächtnis", an index named
Die.Bergretter.S18E05... beside its video wishtv-diebergretter-s18e05.
The par2 merge joined releases only by first-16-KiB hashes, which
readable files never get. A name pass now looks, for every set the
merger finds, at the set poster's releases in its group posted within
12 hours: those whose files the set all names (without case or
punctuation, Latin-1-read UTF-8 read back; sidecars, par2 and images
aside) join the set's release, within the hash pass's file bound.
Merges are kept in release_merges as via 'name'. releases:merge-names
runs it over the sets found before; on 300 recent sets 22 merges of 147
releases, all one upload each.
Also: volumes split after .wmv, .mov, .mpg and other video extensions
share a stem ("Morning Lovers.wmv.001" to ".058" were 58 releases), in
the Rust edge and the TypeScript identity.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69 lines
4.2 KiB
TypeScript
69 lines
4.2 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import BatchStore from '#services/batch_store'
|
|
import { Par2Merger, fileKey } from '#services/par2_merger'
|
|
import { RangeIndexer } from '#lib/edge/index'
|
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
|
import type { Overview } from '#lib/nntp/index'
|
|
|
|
const GROUP = 'alt.binaries.test'
|
|
|
|
/** One file without a counter, from `poster`, posted `minutes` after the first. */
|
|
function file(name: string, at: number, minutes: number, poster = 'up <up@load.er>'): Overview[] {
|
|
const date = new Date(Date.parse('2026-10-20T10:00:00Z') + minutes * 60_000).toUTCString().replace('GMT', '+0000')
|
|
return [[at, { 'subject': `"${name}" yEnc (1/1)`, 'from': poster, 'date': date, 'message-id': `<${at}@test>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at}` }] as Overview]
|
|
}
|
|
|
|
let stores = 0
|
|
async function store(overviews: Overview[]) {
|
|
stores++
|
|
const g = (await db.from('groups').where('name', GROUP).first()) ?? (await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('*'))[0]
|
|
const [n] = await db.table('nodes').insert({ name: `n${stores}` }).returning('id')
|
|
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: stores * 100_000, end_id: stores * 100_000 + 99_999, next_id: stores * 100_000, status: 'leased', node_id: n.id }).returning('id')
|
|
const indexer = new RangeIndexer({ group: GROUP, start: stores * 100_000, end: stores * 100_000 + 99_999, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
|
indexer.add(overviews, stores * 100_000 + 100_000)
|
|
indexer.finish()
|
|
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
|
}
|
|
|
|
test.group('Par2Merger / by file names', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('a name as compared: no case, spaces or punctuation, UTF-8 read as Latin-1 read back', ({ assert }) => {
|
|
assert.equal(fileKey('Some/Dir/Ein.dunkles.Vermächtnis - 1080p.MKV'), fileKey('Ein dunkles Vermächtnis 1080p.mkv'))
|
|
assert.notEqual(fileKey('Track One.flac'), fileKey('Track Two.flac'))
|
|
})
|
|
|
|
test("a set's poster's releases nearby whose files it all names join its release; others stay", async ({ assert }) => {
|
|
await store([
|
|
// The album's index files, its tracks apart (no counter: a release each), and the same poster's next album.
|
|
...file('Some.Album-WEB-2026.par2', 100_010, 0),
|
|
...file('Some.Album-WEB-2026.nfo', 100_011, 0),
|
|
...file('Track One.flac', 100_020, 1),
|
|
...file('Track Two.flac', 100_030, 2),
|
|
...file('Ein.dunkles.Vermächtnis.mkv', 100_040, 3),
|
|
...file('Another Song.flac', 100_050, 4),
|
|
// A track the set names, but posted by someone else, and one a day later.
|
|
...file('Track One.flac', 100_060, 5, 'other <o@x>'),
|
|
])
|
|
await store(file('Track Two.flac', 200_070, 24 * 60 + 10))
|
|
const ids = Object.fromEntries((await db.from('releases').select('id', 'name', 'poster', 'posted_at')).map((r) => [`${r.name}${r.poster.startsWith('other') ? ' (other)' : ''}${new Date(r.posted_at).getUTCDate() === 21 ? ' (later)' : ''}`, Number(r.id)]))
|
|
const index = ids['Some.Album-WEB-2026']
|
|
const setId = 'a'.repeat(32)
|
|
await db.table('par2_files').multiInsert(
|
|
['Track One.flac', 'Track Two.flac', 'Ein.dunkles.Vermächtnis.mkv'].map((name, i) => ({ set_id: setId, file_id: String(i).repeat(32), hash16k: 'f'.repeat(32), length: 600_000, name }))
|
|
)
|
|
await db.table('par2_sets').insert({ set_id: setId, release_id: index, files: 3 })
|
|
|
|
assert.equal(await new Par2Merger().mergeByName(setId), 3)
|
|
const left = (await db.from('releases').orderBy('name').select('name', 'files', 'poster')).map((r) => `${r.name} ${r.files}${r.poster.startsWith('other') ? ' (other)' : ''}`)
|
|
assert.includeMembers(left, ['Another Song 1', 'Track One 1 (other)', 'Track Two 1'])
|
|
assert.lengthOf(left, 4)
|
|
const merged = await db.from('release_merges').where('via', 'name').count('* as n')
|
|
assert.equal(Number(merged[0].n), 3)
|
|
// Nothing more to find.
|
|
assert.equal(await new Par2Merger().mergeByName(setId), 0)
|
|
})
|
|
})
|