Daan MeijerandClaude Opus 5.5
2823d42bfa
hep, hephaestus and gyges post each file under a subject name of its own
(31,000 one-file releases), but the yEnc name in each file's first article
keeps the upload's random stem ("bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001"),
one stem per counter total. The merger's second pass joins releases whose
lone inspections share a random stem: one group and poster, the same total,
within 48 h, no more files than the total plus 10. An expression index over
the stem; release_merges records the kind (via) and takes a stem.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
184 lines
10 KiB
TypeScript
184 lines
10 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import { Par2Merger } from '#services/par2_merger'
|
|
|
|
const hash = (c: string) => c.repeat(32)
|
|
|
|
test.group('Par2Merger', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('merges the one-file releases of an obfuscated upload by its par2 set, named by the par2', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const release = async (key: string, posted: Date, groupId: number, files = 1) => {
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({
|
|
release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: groupId,
|
|
files, segments: files * 10, bytes: files * 1000, first_article: 100, last_article: 200, posted_at: posted, readable: false, file_total: 20,
|
|
settled_at: posted,
|
|
})
|
|
.returning('id')
|
|
return Number(row.id)
|
|
}
|
|
// The par2 index, two data files of the upload, the same file reposted a week later, and one in another group.
|
|
const par2 = await release('14f5999a', at, 1)
|
|
const a = await release('2b7c00e1', new Date(at.getTime() + 60_000), 1)
|
|
const b = await release('9d01aa3f', new Date(at.getTime() + 120_000), 1, 2)
|
|
const later = await release('5e5e5e5e', new Date(at.getTime() + 7 * 86400_000), 1)
|
|
const elsewhere = await release('6f6f6f6f', at, 2)
|
|
let candidate = 1000
|
|
const hashed = async (releaseId: number, h: string, size: number) => {
|
|
candidate++
|
|
await db.table('probe_candidates').insert({ id: candidate, release_id: releaseId, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
|
|
await db.table('file_hashes').insert({ candidate_id: candidate, release_id: releaseId, hash16k: h, size })
|
|
}
|
|
await hashed(a, hash('a'), 104857600)
|
|
await hashed(b, hash('b'), 104857600)
|
|
await hashed(later, hash('a'), 104857600)
|
|
await hashed(elsewhere, hash('b'), 104857600)
|
|
// A different size: not the same file.
|
|
await hashed(await release('7a7a7a7a', at, 1), hash('a'), 5)
|
|
const setId = 'c'.repeat(32)
|
|
await db.table('par2_files').multiInsert([
|
|
{ set_id: setId, file_id: 'd'.repeat(32), hash16k: hash('a'), length: 104857600, name: 'Some.Movie.2022.1080p.part01.rar' },
|
|
{ set_id: setId, file_id: 'e'.repeat(32), hash16k: hash('b'), length: 104857600, name: 'Some.Movie.2022.1080p.part02.rar' },
|
|
])
|
|
await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 })
|
|
|
|
const round = await new Par2Merger().round()
|
|
assert.deepEqual(round, { sets: 1, merges: 1, releases: 3, stems: 0, stemMerges: 0, stemReleases: 0 })
|
|
const target = await db.from('releases').where('id', b).firstOrFail()
|
|
assert.equal(target.files, 4)
|
|
assert.equal(target.segments, 40)
|
|
assert.equal(target.name, 'Some.Movie.2022.1080p')
|
|
assert.isTrue(target.complete === false)
|
|
assert.deepEqual((await db.from('releases').whereIn('id', [par2, a]).select('id')).length, 0)
|
|
assert.equal((await db.from('releases').whereIn('id', [later, elsewhere]).select('id')).length, 2)
|
|
assert.deepEqual((await db.from('probe_candidates').where('release_id', b).orderBy('id').select('id')).map((r) => Number(r.id)), [1001, 1002])
|
|
assert.equal((await db.from('par2_sets').where('set_id', setId).firstOrFail()).release_id, String(b))
|
|
assert.equal((await db.from('release_merges').where('into_id', b).count('* as n'))[0].n, '2')
|
|
|
|
// Again: nothing new.
|
|
assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0)
|
|
})
|
|
|
|
test('a set held by two of the releases merged is held once by the target', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const ids: number[] = []
|
|
for (const [key, files] of [['bb000001', 1], ['bb000002', 1], ['bb000003', 3]] as const) {
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files, posted_at: at, readable: false })
|
|
.returning('id')
|
|
ids.push(Number(row.id))
|
|
}
|
|
const setId = hash('7')
|
|
// The par2 index and its repost each hold the set; the third release holds the data file.
|
|
await db.table('par2_sets').multiInsert([
|
|
{ set_id: setId, release_id: ids[0], files: 1 },
|
|
{ set_id: setId, release_id: ids[1], files: 1 },
|
|
])
|
|
await db.table('par2_files').insert({ set_id: setId, file_id: hash('6'), hash16k: hash('5'), length: 1000, name: 'x.rar' })
|
|
await db.table('file_hashes').insert({ candidate_id: 3001, release_id: ids[2], hash16k: hash('5'), size: 1000 })
|
|
assert.equal(await new Par2Merger().mergeSet(setId), 2)
|
|
const held = await db.from('par2_sets').where('set_id', setId).select('release_id')
|
|
assert.deepEqual(held.map((r) => Number(r.release_id)), [ids[2]])
|
|
})
|
|
|
|
test('a set held by many releases nearby is posted again and again: not merged', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const setId = hash('4')
|
|
const ids: number[] = []
|
|
for (let i = 1; i <= 6; i++) {
|
|
const key = `cc00000${i}`
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `kantodo${i}`, source: 'stem', groups: '[]', group_id: 1, files: 11, posted_at: at, readable: true })
|
|
.returning('id')
|
|
ids.push(Number(row.id))
|
|
await db.table('par2_sets').insert({ set_id: setId, release_id: row.id, files: 1 })
|
|
}
|
|
await db.table('par2_files').insert({ set_id: setId, file_id: hash('3'), hash16k: hash('2'), length: 1073741824, name: 'kantodo.bin' })
|
|
assert.equal(await new Par2Merger().mergeSet(setId), 0)
|
|
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 6)
|
|
})
|
|
|
|
test('merges the one-file releases of an obfuscated upload by the random stem of their inner names', async ({ assert }) => {
|
|
const at = new Date('2022-07-07T19:34:00Z')
|
|
let n = 0
|
|
const release = async (poster: string, posted: Date, total: number, inner: string | null, groupId = 1) => {
|
|
n++
|
|
const key = `dd${String(n).padStart(6, '0')}`
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `hex${n}`, source: 'stem', groups: '[]', group_id: groupId, files: 1, posted_at: posted, readable: false, file_total: total, poster })
|
|
.returning('id')
|
|
if (inner) {
|
|
await db.table('release_inspections').insert({ release_id: row.id, candidate_id: 5000 + n, kind: 'lone', result: JSON.stringify({ yencName: inner, type: '', size: 1 }) })
|
|
}
|
|
return Number(row.id)
|
|
}
|
|
const hep = 'hep <adsjfhsdh@sdljf>'
|
|
const stem = 'nabNGjSlKhOYJCslnEw1MzJIgPIjffyT'
|
|
const a = await release(hep, at, 15, `${stem}.7z.001`)
|
|
const b = await release(hep, new Date(at.getTime() + 60_000), 15, `${stem}.7z.002`)
|
|
const c = await release(hep, new Date(at.getTime() + 120_000), 15, `${stem}.vol07+08.par2`)
|
|
// Another poster, a week later, another total, another group: apart.
|
|
const other = await release('someone <x@y>', at, 15, `${stem}.7z.003`)
|
|
const later = await release(hep, new Date(at.getTime() + 7 * 86400_000), 15, `${stem}.7z.004`)
|
|
const total = await release(hep, at, 16, `${stem}.7z.005`)
|
|
const group = await release(hep, at, 15, `${stem}.7z.006`, 2)
|
|
// A readable stem isn't merged on: a repost would share it.
|
|
const r1 = await release(hep, at, 15, 'SomeReadableMovieName.7z.001')
|
|
const r2 = await release(hep, at, 15, 'SomeReadableMovieName.7z.002')
|
|
|
|
const round = await new Par2Merger().round()
|
|
assert.equal(round.stemMerges, 1)
|
|
assert.equal(round.stemReleases, 3)
|
|
const left = (await db.from('releases').whereIn('id', [a, b, c]).select('id', 'files')).map((r) => Number(r.files))
|
|
assert.deepEqual(left, [3])
|
|
assert.equal((await db.from('releases').whereIn('id', [other, later, total, group, r1, r2]).select('id')).length, 6)
|
|
const merges = await db.from('release_merges').where('via', 'inner').select('set_id')
|
|
assert.deepEqual([...new Set(merges.map((m) => m.set_id))], [stem])
|
|
})
|
|
|
|
test('an inner-name merge with more files than the counter is more than one upload', async ({ assert }) => {
|
|
const at = new Date('2022-07-07T19:34:00Z')
|
|
const stem = 'R8aUMXqWQfOaei4u74fch9v0skxUY7ah'
|
|
const ids: number[] = []
|
|
for (let i = 1; i <= 3; i++) {
|
|
const key = `ee00000${i}`
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 10, posted_at: at, readable: false, file_total: 5, poster: 'p' })
|
|
.returning('id')
|
|
ids.push(Number(row.id))
|
|
await db.table('release_inspections').insert({ release_id: row.id, candidate_id: 6000 + i, kind: 'lone', result: JSON.stringify({ yencName: `${stem}.7z.00${i}` }) })
|
|
}
|
|
assert.equal(await new Par2Merger().mergeStem(stem), 0)
|
|
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3)
|
|
})
|
|
|
|
test('a set file that matches several releases nearby is ambiguous: not merged', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const ids: number[] = []
|
|
for (const key of ['aa000001', 'aa000002', 'aa000003']) {
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false })
|
|
.returning('id')
|
|
ids.push(Number(row.id))
|
|
}
|
|
// Two test uploads whose 1 GiB files start alike, and a set describing one such file, held by the third.
|
|
await db.table('file_hashes').multiInsert([
|
|
{ candidate_id: 2001, release_id: ids[0], hash16k: hash('f'), size: 1073741824 },
|
|
{ candidate_id: 2002, release_id: ids[1], hash16k: hash('f'), size: 1073741824 },
|
|
])
|
|
await db.table('par2_files').insert({ set_id: hash('9'), file_id: hash('8'), hash16k: hash('f'), length: 1073741824, name: 'kantibanito1.bin' })
|
|
await db.table('par2_sets').insert({ set_id: hash('9'), release_id: ids[2], files: 1 })
|
|
assert.equal((await new Par2Merger().round()).merges, 0)
|
|
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3)
|
|
})
|
|
})
|