From 6d1f2cc2a192508bb6b307088015a9c4523532c4 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 12 Oct 2026 10:15:32 +0200 Subject: [PATCH] Par2 merge: a set file matching several releases nearby is ambiguous 131 uploads of 1 GiB test files ("kantodoNNN.bin", a.b.test) share their first 16 KiB and size, so all matched one set's single file and became one release of 1,284 files. A par2 file that matches more than one release in the anchor's group and time window is left out; a plain repost of the same file stays apart too. Co-Authored-By: Claude Opus 5.5 --- app/services/par2_merger.ts | 40 +++++++++++++++++++--------- tests/functional/par2_merger.spec.ts | 21 +++++++++++++++ 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/app/services/par2_merger.ts b/app/services/par2_merger.ts index 2cc174d..192c2b8 100644 --- a/app/services/par2_merger.ts +++ b/app/services/par2_merger.ts @@ -85,21 +85,37 @@ export class Par2Merger { /** Merges the releases of one set; how many went into another. */ async mergeSet(setId: string): Promise { return db.transaction(async (trx) => { - const { rows } = await trx.rawQuery( - `WITH m AS ( - SELECT release_id, true AS holds FROM par2_sets WHERE set_id = ? - UNION ALL - SELECT h.release_id, false FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length WHERE f.set_id = ? - ), d AS (SELECT release_id, bool_or(holds) AS holds FROM m GROUP BY 1) - SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, d.holds - FROM releases r JOIN d ON d.release_id = r.id ORDER BY r.id FOR UPDATE OF r`, - [setId, setId] + const { rows: holders } = await trx.rawQuery( + `SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id + WHERE s.set_id = ? ORDER BY r.id`, + [setId] ) - const anchor = rows.find((r: Member & { holds: boolean }) => r.holds) + const anchor: Member | undefined = holders[0] if (!anchor) return 0 const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null) - const near = (r: Member) => at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS - const members: Member[] = rows.filter((r: Member) => r.group_id === anchor.group_id && near(r)) + const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS) + const { rows: matches } = await trx.rawQuery( + `SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable + FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id + WHERE f.set_id = ?`, + [setId] + ) + // A file of the set that matches more than one release nearby is ambiguous: files that share their first + // 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became + // one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too. + const byFile = new Map>() + for (const m of matches.filter(near)) { + const releases = byFile.get(m.file_id) ?? new Map() + releases.set(String(m.id), m) + byFile.set(m.file_id, releases) + } + const found = new Map() + for (const r of holders.filter(near)) found.set(String(r.id), r) + for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r) + if (found.size < 2) return 0 + // Locked in id order (the batch store and other rounds take the same rows). + const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]]) + const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!) if (members.length < 2) return 0 const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a)) const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id)) diff --git a/tests/functional/par2_merger.spec.ts b/tests/functional/par2_merger.spec.ts index c24a2b6..8ec420d 100644 --- a/tests/functional/par2_merger.spec.ts +++ b/tests/functional/par2_merger.spec.ts @@ -62,4 +62,25 @@ test.group('Par2Merger', (group) => { // Again: nothing new. assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0) }) + + test('a set file that matches several releases nearby is ambiguous: not merged', async ({ assert }) => { + const at = new Date('2022-07-08T03:46:00Z') + const ids: number[] = [] + for (const key of ['aa000001', 'aa000002', 'aa000003']) { + const [row] = await db + .table('releases') + .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false }) + .returning('id') + ids.push(Number(row.id)) + } + // Two test uploads whose 1 GiB files start alike, and a set describing one such file, held by the third. + await db.table('file_hashes').multiInsert([ + { candidate_id: 2001, release_id: ids[0], hash16k: hash('f'), size: 1073741824 }, + { candidate_id: 2002, release_id: ids[1], hash16k: hash('f'), size: 1073741824 }, + ]) + await db.table('par2_files').insert({ set_id: hash('9'), file_id: hash('8'), hash16k: hash('f'), length: 1073741824, name: 'kantibanito1.bin' }) + await db.table('par2_sets').insert({ set_id: hash('9'), release_id: ids[2], files: 1 }) + assert.equal((await new Par2Merger().round()).merges, 0) + assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3) + }) })