import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import { Par2Merger } from '#services/par2_merger' const hash = (c: string) => c.repeat(32) test.group('Par2Merger', (group) => { group.each.setup(() => testUtils.db().withGlobalTransaction()) test('merges the one-file releases of an obfuscated upload by its par2 set, named by the par2', async ({ assert }) => { const at = new Date('2022-07-08T03:46:00Z') const release = async (key: string, posted: Date, groupId: number, files = 1) => { const [row] = await db .table('releases') .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: groupId, files, segments: files * 10, bytes: files * 1000, first_article: 100, last_article: 200, posted_at: posted, readable: false, file_total: 20, settled_at: posted, }) .returning('id') return Number(row.id) } // The par2 index, two data files of the upload, the same file reposted a week later, and one in another group. const par2 = await release('14f5999a', at, 1) const a = await release('2b7c00e1', new Date(at.getTime() + 60_000), 1) const b = await release('9d01aa3f', new Date(at.getTime() + 120_000), 1, 2) const later = await release('5e5e5e5e', new Date(at.getTime() + 7 * 86400_000), 1) const elsewhere = await release('6f6f6f6f', at, 2) let candidate = 1000 const hashed = async (releaseId: number, h: string, size: number) => { candidate++ await db.table('probe_candidates').insert({ id: candidate, release_id: releaseId, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' }) await db.table('file_hashes').insert({ candidate_id: candidate, release_id: releaseId, hash16k: h, size }) } await hashed(a, hash('a'), 104857600) await hashed(b, hash('b'), 104857600) await hashed(later, hash('a'), 104857600) await hashed(elsewhere, hash('b'), 104857600) // A different size: not the same file. await hashed(await release('7a7a7a7a', at, 1), hash('a'), 5) const setId = 'c'.repeat(32) await db.table('par2_files').multiInsert([ { set_id: setId, file_id: 'd'.repeat(32), hash16k: hash('a'), length: 104857600, name: 'Some.Movie.2022.1080p.part01.rar' }, { set_id: setId, file_id: 'e'.repeat(32), hash16k: hash('b'), length: 104857600, name: 'Some.Movie.2022.1080p.part02.rar' }, ]) await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 }) const round = await new Par2Merger().round() assert.deepEqual(round, { sets: 1, merges: 1, releases: 3, stems: 0, stemMerges: 0, stemReleases: 0 }) const target = await db.from('releases').where('id', b).firstOrFail() assert.equal(target.files, 4) assert.equal(target.segments, 40) assert.equal(target.name, 'Some.Movie.2022.1080p') assert.isTrue(target.complete === false) assert.deepEqual((await db.from('releases').whereIn('id', [par2, a]).select('id')).length, 0) assert.equal((await db.from('releases').whereIn('id', [later, elsewhere]).select('id')).length, 2) assert.deepEqual((await db.from('probe_candidates').where('release_id', b).orderBy('id').select('id')).map((r) => Number(r.id)), [1001, 1002]) assert.equal((await db.from('par2_sets').where('set_id', setId).firstOrFail()).release_id, String(b)) assert.equal((await db.from('release_merges').where('into_id', b).count('* as n'))[0].n, '2') // Again: nothing new. assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0) }) test('a set held by two of the releases merged is held once by the target', async ({ assert }) => { const at = new Date('2022-07-08T03:46:00Z') const ids: number[] = [] for (const [key, files] of [['bb000001', 1], ['bb000002', 1], ['bb000003', 3]] as const) { const [row] = await db .table('releases') .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files, posted_at: at, readable: false }) .returning('id') ids.push(Number(row.id)) } const setId = hash('7') // The par2 index and its repost each hold the set; the third release holds the data file. await db.table('par2_sets').multiInsert([ { set_id: setId, release_id: ids[0], files: 1 }, { set_id: setId, release_id: ids[1], files: 1 }, ]) await db.table('par2_files').insert({ set_id: setId, file_id: hash('6'), hash16k: hash('5'), length: 1000, name: 'x.rar' }) await db.table('file_hashes').insert({ candidate_id: 3001, release_id: ids[2], hash16k: hash('5'), size: 1000 }) assert.equal(await new Par2Merger().mergeSet(setId), 2) const held = await db.from('par2_sets').where('set_id', setId).select('release_id') assert.deepEqual(held.map((r) => Number(r.release_id)), [ids[2]]) }) test('a set held by many releases nearby is posted again and again: not merged', async ({ assert }) => { const at = new Date('2022-07-08T03:46:00Z') const setId = hash('4') const ids: number[] = [] for (let i = 1; i <= 6; i++) { const key = `cc00000${i}` const [row] = await db .table('releases') .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `kantodo${i}`, source: 'stem', groups: '[]', group_id: 1, files: 11, posted_at: at, readable: true }) .returning('id') ids.push(Number(row.id)) await db.table('par2_sets').insert({ set_id: setId, release_id: row.id, files: 1 }) } await db.table('par2_files').insert({ set_id: setId, file_id: hash('3'), hash16k: hash('2'), length: 1073741824, name: 'kantodo.bin' }) assert.equal(await new Par2Merger().mergeSet(setId), 0) assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 6) }) test('merges the one-file releases of an obfuscated upload by the random stem of their inner names', async ({ assert }) => { const at = new Date('2022-07-07T19:34:00Z') let n = 0 const release = async (poster: string, posted: Date, total: number, inner: string | null, groupId = 1) => { n++ const key = `dd${String(n).padStart(6, '0')}` const [row] = await db .table('releases') .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `hex${n}`, source: 'stem', groups: '[]', group_id: groupId, files: 1, posted_at: posted, readable: false, file_total: total, poster }) .returning('id') if (inner) { await db.table('release_inspections').insert({ release_id: row.id, candidate_id: 5000 + n, kind: 'lone', result: JSON.stringify({ yencName: inner, type: '', size: 1 }) }) } return Number(row.id) } const hep = 'hep ' const stem = 'nabNGjSlKhOYJCslnEw1MzJIgPIjffyT' const a = await release(hep, at, 15, `${stem}.7z.001`) const b = await release(hep, new Date(at.getTime() + 60_000), 15, `${stem}.7z.002`) const c = await release(hep, new Date(at.getTime() + 120_000), 15, `${stem}.vol07+08.par2`) // Another poster, a week later, another total, another group: apart. const other = await release('someone ', at, 15, `${stem}.7z.003`) const later = await release(hep, new Date(at.getTime() + 7 * 86400_000), 15, `${stem}.7z.004`) const total = await release(hep, at, 16, `${stem}.7z.005`) const group = await release(hep, at, 15, `${stem}.7z.006`, 2) // A readable stem isn't merged on: a repost would share it. const r1 = await release(hep, at, 15, 'SomeReadableMovieName.7z.001') const r2 = await release(hep, at, 15, 'SomeReadableMovieName.7z.002') const round = await new Par2Merger().round() assert.equal(round.stemMerges, 1) assert.equal(round.stemReleases, 3) const left = (await db.from('releases').whereIn('id', [a, b, c]).select('id', 'files')).map((r) => Number(r.files)) assert.deepEqual(left, [3]) assert.equal((await db.from('releases').whereIn('id', [other, later, total, group, r1, r2]).select('id')).length, 6) const merges = await db.from('release_merges').where('via', 'inner').select('set_id') assert.deepEqual([...new Set(merges.map((m) => m.set_id))], [stem]) }) test('an inner-name merge with more files than the counter is more than one upload', async ({ assert }) => { const at = new Date('2022-07-07T19:34:00Z') const stem = 'R8aUMXqWQfOaei4u74fch9v0skxUY7ah' const ids: number[] = [] for (let i = 1; i <= 3; i++) { const key = `ee00000${i}` const [row] = await db .table('releases') .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 10, posted_at: at, readable: false, file_total: 5, poster: 'p' }) .returning('id') ids.push(Number(row.id)) await db.table('release_inspections').insert({ release_id: row.id, candidate_id: 6000 + i, kind: 'lone', result: JSON.stringify({ yencName: `${stem}.7z.00${i}` }) }) } assert.equal(await new Par2Merger().mergeStem(stem), 0) assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3) }) test('a set file that matches several releases nearby is ambiguous: not merged', async ({ assert }) => { const at = new Date('2022-07-08T03:46:00Z') const ids: number[] = [] for (const key of ['aa000001', 'aa000002', 'aa000003']) { const [row] = await db .table('releases') .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false }) .returning('id') ids.push(Number(row.id)) } // Two test uploads whose 1 GiB files start alike, and a set describing one such file, held by the third. await db.table('file_hashes').multiInsert([ { candidate_id: 2001, release_id: ids[0], hash16k: hash('f'), size: 1073741824 }, { candidate_id: 2002, release_id: ids[1], hash16k: hash('f'), size: 1073741824 }, ]) await db.table('par2_files').insert({ set_id: hash('9'), file_id: hash('8'), hash16k: hash('f'), length: 1073741824, name: 'kantibanito1.bin' }) await db.table('par2_sets').insert({ set_id: hash('9'), release_id: ids[2], files: 1 }) assert.equal((await new Par2Merger().round()).merges, 0) assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3) }) })