Merger: what's new an hour at a time, only the sets two releases share; common files match nothing
A round behind by two days of the probe backlog read 3.5M sets and never ended; a par2 file shared by up to 190k files loaded every release it matched and the merger ran out of heap every few minutes (LOG.md 2026-10-20 (85)). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9725bdbd7b
commit
4f41303482
3 files changed
+153
-46
No files matched your search
@@ -78,6 +78,44 @@ test.group('Par2Merger', (group) => {
|
||||
assert.equal((await new Par2Merger().round()).sets, 1)
|
||||
})
|
||||
|
||||
test('a round behind by hours goes through them a window at a time, past the gaps', async ({ assert }) => {
|
||||
const ago = (hours: number) => new Date(Date.now() - hours * 3600_000)
|
||||
await db.table('par2_sets').multiInsert([
|
||||
{ set_id: 'a1'.repeat(16), release_id: 1, files: 1, created_at: ago(30) },
|
||||
{ set_id: 'a2'.repeat(16), release_id: 2, files: 1, created_at: ago(2.5) },
|
||||
{ set_id: 'a3'.repeat(16), release_id: 3, files: 1, created_at: ago(0.5) },
|
||||
])
|
||||
await db.table('worker_cursors').insert({ name: 'merger', at: ago(48) })
|
||||
assert.equal((await new Par2Merger().round()).sets, 3)
|
||||
const saved = await db.from('worker_cursors').where('name', 'merger').firstOrFail()
|
||||
assert.isAbove(new Date(saved.at).getTime(), ago(0.5).getTime())
|
||||
})
|
||||
|
||||
test('a par2 file held by more than 20 files is a common one: it merges nothing', async ({ assert }) => {
|
||||
const at = new Date('2022-07-08T03:46:00Z')
|
||||
const ids: number[] = []
|
||||
for (const key of ['dd000001', 'dd000002']) {
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false })
|
||||
.returning('id')
|
||||
ids.push(Number(row.id))
|
||||
}
|
||||
// The par2 is held by the first; the second holds the file, and so do 20 releases elsewhere and years apart.
|
||||
const setId = hash('1')
|
||||
await db.table('par2_sets').insert({ set_id: setId, release_id: ids[0], files: 1 })
|
||||
await db.table('par2_files').insert({ set_id: setId, file_id: hash('0'), hash16k: hash('9'), length: 30214, name: 'sample.nfo' })
|
||||
await db.table('file_hashes').insert({ candidate_id: 4000, release_id: ids[1], hash16k: hash('9'), size: 30214 })
|
||||
for (let i = 1; i <= 20; i++) await db.table('file_hashes').insert({ candidate_id: 4000 + i, release_id: 900000 + i, hash16k: hash('9'), size: 30214 })
|
||||
const round = await new Par2Merger().round()
|
||||
assert.deepEqual([round.sets, round.merges], [1, 0])
|
||||
assert.equal(await new Par2Merger().mergeSet(setId), 0)
|
||||
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 2)
|
||||
// One holder fewer: the file is one upload's again.
|
||||
await db.from('file_hashes').where('candidate_id', 4020).delete()
|
||||
assert.equal(await new Par2Merger().mergeSet(setId), 1)
|
||||
})
|
||||
|
||||
test('a set held by two of the releases merged is held once by the target', async ({ assert }) => {
|
||||
const at = new Date('2022-07-08T03:46:00Z')
|
||||
const ids: number[] = []
|
||||
|
||||
Reference in new issue
Block a user