Daan MeijerandClaude Opus 5.5
c72bf68fd2
A par2 index and its repost both hold a set: moving both rows to the target broke par2_sets' primary key, and the round failed on it every minute from 20:28 UTC (no merges at all). Duplicates among the others go first; a set that fails is logged and the round goes on. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
6.1 KiB
TypeScript
110 lines
6.1 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import { Par2Merger } from '#services/par2_merger'
|
|
|
|
const hash = (c: string) => c.repeat(32)
|
|
|
|
test.group('Par2Merger', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('merges the one-file releases of an obfuscated upload by its par2 set, named by the par2', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const release = async (key: string, posted: Date, groupId: number, files = 1) => {
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({
|
|
release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: groupId,
|
|
files, segments: files * 10, bytes: files * 1000, first_article: 100, last_article: 200, posted_at: posted, readable: false, file_total: 20,
|
|
settled_at: posted,
|
|
})
|
|
.returning('id')
|
|
return Number(row.id)
|
|
}
|
|
// The par2 index, two data files of the upload, the same file reposted a week later, and one in another group.
|
|
const par2 = await release('14f5999a', at, 1)
|
|
const a = await release('2b7c00e1', new Date(at.getTime() + 60_000), 1)
|
|
const b = await release('9d01aa3f', new Date(at.getTime() + 120_000), 1, 2)
|
|
const later = await release('5e5e5e5e', new Date(at.getTime() + 7 * 86400_000), 1)
|
|
const elsewhere = await release('6f6f6f6f', at, 2)
|
|
let candidate = 1000
|
|
const hashed = async (releaseId: number, h: string, size: number) => {
|
|
candidate++
|
|
await db.table('probe_candidates').insert({ id: candidate, release_id: releaseId, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
|
|
await db.table('file_hashes').insert({ candidate_id: candidate, release_id: releaseId, hash16k: h, size })
|
|
}
|
|
await hashed(a, hash('a'), 104857600)
|
|
await hashed(b, hash('b'), 104857600)
|
|
await hashed(later, hash('a'), 104857600)
|
|
await hashed(elsewhere, hash('b'), 104857600)
|
|
// A different size: not the same file.
|
|
await hashed(await release('7a7a7a7a', at, 1), hash('a'), 5)
|
|
const setId = 'c'.repeat(32)
|
|
await db.table('par2_files').multiInsert([
|
|
{ set_id: setId, file_id: 'd'.repeat(32), hash16k: hash('a'), length: 104857600, name: 'Some.Movie.2022.1080p.part01.rar' },
|
|
{ set_id: setId, file_id: 'e'.repeat(32), hash16k: hash('b'), length: 104857600, name: 'Some.Movie.2022.1080p.part02.rar' },
|
|
])
|
|
await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 })
|
|
|
|
const round = await new Par2Merger().round()
|
|
assert.deepEqual(round, { sets: 1, merges: 1, releases: 3 })
|
|
const target = await db.from('releases').where('id', b).firstOrFail()
|
|
assert.equal(target.files, 4)
|
|
assert.equal(target.segments, 40)
|
|
assert.equal(target.name, 'Some.Movie.2022.1080p')
|
|
assert.isTrue(target.complete === false)
|
|
assert.deepEqual((await db.from('releases').whereIn('id', [par2, a]).select('id')).length, 0)
|
|
assert.equal((await db.from('releases').whereIn('id', [later, elsewhere]).select('id')).length, 2)
|
|
assert.deepEqual((await db.from('probe_candidates').where('release_id', b).orderBy('id').select('id')).map((r) => Number(r.id)), [1001, 1002])
|
|
assert.equal((await db.from('par2_sets').where('set_id', setId).firstOrFail()).release_id, String(b))
|
|
assert.equal((await db.from('release_merges').where('into_id', b).count('* as n'))[0].n, '2')
|
|
|
|
// Again: nothing new.
|
|
assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0)
|
|
})
|
|
|
|
test('a set held by two of the releases merged is held once by the target', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const ids: number[] = []
|
|
for (const [key, files] of [['bb000001', 1], ['bb000002', 1], ['bb000003', 3]] as const) {
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files, posted_at: at, readable: false })
|
|
.returning('id')
|
|
ids.push(Number(row.id))
|
|
}
|
|
const setId = hash('7')
|
|
// The par2 index and its repost each hold the set; the third release holds the data file.
|
|
await db.table('par2_sets').multiInsert([
|
|
{ set_id: setId, release_id: ids[0], files: 1 },
|
|
{ set_id: setId, release_id: ids[1], files: 1 },
|
|
])
|
|
await db.table('par2_files').insert({ set_id: setId, file_id: hash('6'), hash16k: hash('5'), length: 1000, name: 'x.rar' })
|
|
await db.table('file_hashes').insert({ candidate_id: 3001, release_id: ids[2], hash16k: hash('5'), size: 1000 })
|
|
assert.equal(await new Par2Merger().mergeSet(setId), 2)
|
|
const held = await db.from('par2_sets').where('set_id', setId).select('release_id')
|
|
assert.deepEqual(held.map((r) => Number(r.release_id)), [ids[2]])
|
|
})
|
|
|
|
test('a set file that matches several releases nearby is ambiguous: not merged', async ({ assert }) => {
|
|
const at = new Date('2022-07-08T03:46:00Z')
|
|
const ids: number[] = []
|
|
for (const key of ['aa000001', 'aa000002', 'aa000003']) {
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false })
|
|
.returning('id')
|
|
ids.push(Number(row.id))
|
|
}
|
|
// Two test uploads whose 1 GiB files start alike, and a set describing one such file, held by the third.
|
|
await db.table('file_hashes').multiInsert([
|
|
{ candidate_id: 2001, release_id: ids[0], hash16k: hash('f'), size: 1073741824 },
|
|
{ candidate_id: 2002, release_id: ids[1], hash16k: hash('f'), size: 1073741824 },
|
|
])
|
|
await db.table('par2_files').insert({ set_id: hash('9'), file_id: hash('8'), hash16k: hash('f'), length: 1073741824, name: 'kantibanito1.bin' })
|
|
await db.table('par2_sets').insert({ set_id: hash('9'), release_id: ids[2], files: 1 })
|
|
assert.equal((await new Par2Merger().round()).merges, 0)
|
|
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3)
|
|
})
|
|
})
|