diff --git a/app/services/par2_merger.ts b/app/services/par2_merger.ts index 9822280..f531399 100644 --- a/app/services/par2_merger.ts +++ b/app/services/par2_merger.ts @@ -5,6 +5,14 @@ import { looksRandom } from '#lib/release/random' /** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */ const MAX_SPREAD_MS = 48 * 3600 * 1000 +/** + * A set held by more releases nearby than this is one posted again and again, not one upload's par2 (every + * "kantodoNNN.bin" test upload carries the same set: 64 releases became one of 631 files; LOG.md 2026-10-12 (30)). + */ +const MAX_HOLDERS = 4 +/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */ +const MAX_FILES_PER_SET_FILE = 3 +const SLACK_FILES = 10 /** Between rounds. */ const ROUND_MS = 60_000 /** A round looks back this far past the previous one's start (rows committed while it ran). */ @@ -113,14 +121,19 @@ export class Par2Merger { releases.set(String(m.id), m) byFile.set(m.file_id, releases) } + const nearHolders = holders.filter(near) + if (nearHolders.length > MAX_HOLDERS) return 0 const found = new Map() - for (const r of holders.filter(near)) found.set(String(r.id), r) + for (const r of nearHolders) found.set(String(r.id), r) for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r) if (found.size < 2) return 0 // Locked in id order (the batch store and other rounds take the same rows). const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]]) const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!) if (members.length < 2) return 0 + const { rows: described } = await trx.rawQuery(`SELECT count(*) AS n FROM par2_files WHERE set_id = ?`, [setId]) + const setFiles = Math.max(Number(described[0].n), 1) + if (members.reduce((n, m) => n + Number(m.files), 0) > MAX_FILES_PER_SET_FILE * setFiles + SLACK_FILES) return 0 const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a)) const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id)) const name = members.every((m) => !m.readable) ? await this.setName(trx, setId) : undefined diff --git a/tests/functional/par2_merger.spec.ts b/tests/functional/par2_merger.spec.ts index 4c1db80..20ffa14 100644 --- a/tests/functional/par2_merger.spec.ts +++ b/tests/functional/par2_merger.spec.ts @@ -86,6 +86,24 @@ test.group('Par2Merger', (group) => { assert.deepEqual(held.map((r) => Number(r.release_id)), [ids[2]]) }) + test('a set held by many releases nearby is posted again and again: not merged', async ({ assert }) => { + const at = new Date('2022-07-08T03:46:00Z') + const setId = hash('4') + const ids: number[] = [] + for (let i = 1; i <= 6; i++) { + const key = `cc00000${i}` + const [row] = await db + .table('releases') + .insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `kantodo${i}`, source: 'stem', groups: '[]', group_id: 1, files: 11, posted_at: at, readable: true }) + .returning('id') + ids.push(Number(row.id)) + await db.table('par2_sets').insert({ set_id: setId, release_id: row.id, files: 1 }) + } + await db.table('par2_files').insert({ set_id: setId, file_id: hash('3'), hash16k: hash('2'), length: 1073741824, name: 'kantodo.bin' }) + assert.equal(await new Par2Merger().mergeSet(setId), 0) + assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 6) + }) + test('a set file that matches several releases nearby is ambiguous: not merged', async ({ assert }) => { const at = new Date('2022-07-08T03:46:00Z') const ids: number[] = []