From 1a20fcd2c388b7fdc890ebcc67b0e1000dea348d Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 16 Oct 2026 07:59:40 +0200 Subject: [PATCH] Prune par2_files of complete and judged sets (hourly, from the merger) Co-Authored-By: Claude Opus 5.5 --- app/services/par2_merger.ts | 10 ++++ app/services/par2_pruner.ts | 81 ++++++++++++++++++++++++++++ tests/functional/par2_pruner.spec.ts | 68 +++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 app/services/par2_pruner.ts create mode 100644 tests/functional/par2_pruner.spec.ts diff --git a/app/services/par2_merger.ts b/app/services/par2_merger.ts index 073c7c0..d0c899d 100644 --- a/app/services/par2_merger.ts +++ b/app/services/par2_merger.ts @@ -3,6 +3,7 @@ import logger from '@adonisjs/core/services/logger' import { par2ReleaseName } from '#lib/release/index' import { looksRandom } from '#lib/release/random' import { SidecarFolder } from '#services/sidecar_folder' +import { Par2Pruner } from '#services/par2_pruner' /** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */ const MAX_SPREAD_MS = 48 * 3600 * 1000 @@ -20,6 +21,8 @@ const ROUND_MS = 60_000 const OVERLAP_MS = 60_000 /** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */ const FOLD_EVERY_MS = 5 * 60_000 +/** Between par2_files prunes (complete and judged sets' file lists). */ +const PRUNE_EVERY_MS = 60 * 60_000 /** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */ const CURSOR = 'merger' @@ -85,9 +88,16 @@ export class Par2Merger { } async run() { + let prunedAt = 0 while (!this.stopped) { const started = Date.now() try { + // First, at start too: a prune needn't wait for a long round. + if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) { + prunedAt = Date.now() + const pruned = await new Par2Pruner().round() + if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`) + } const round = await this.round() if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`) if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`) diff --git a/app/services/par2_pruner.ts b/app/services/par2_pruner.ts new file mode 100644 index 0000000..b12b6c5 --- /dev/null +++ b/app/services/par2_pruner.ts @@ -0,0 +1,81 @@ +import db from '@adonisjs/lucid/services/db' + +/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */ +const SETTLED_MS = 2 * 86400_000 +/** Sets per statement. */ +const BATCH = 20_000 +/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */ +const CURSOR = 'par2-prune' +/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */ +const JUNK = ['dead', 'fake', 'repeated'] + +export interface PruneRound { + complete: number + judged: number + rows: number +} + +/** + * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and + * 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's + * done when + * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the + * set (all of it merged into one); + * - judged: every release holding it has a junk verdict (dead, fake, repeated). + * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its + * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds. + */ +export class Par2Pruner { + async round(): Promise { + const round: PruneRound = { complete: 0, judged: 0, rows: 0 } + const saved = await db.from('worker_cursors').where('name', CURSOR).first() + let since: Date = saved ? new Date(saved.at) : new Date(0) + const until = new Date(Date.now() - SETTLED_MS) + for (;;) { + const { rows: sets } = await db.rawQuery( + `SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id + HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`, + [since, until, BATCH] + ) + if (sets.length === 0) break + const ids = sets.map((s: { set_id: string }) => s.set_id) + const { rows: done } = await db.rawQuery( + `SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id) + WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id) + AND NOT EXISTS ( + SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS ( + SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id + WHERE h.hash16k = f.hash16k AND h.size = f.length))`, + [ids] + ) + round.complete += done.length + round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id)) + since = sets.at(-1).at + await db.rawQuery( + `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`, + [CURSOR, since] + ) + if (sets.length < BATCH) break + } + // Judged: no cursor (verdicts come and go); only sets with rows left. + const { rows: judged } = await db.rawQuery( + `SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id + WHERE v.rule = ANY(?) + AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id) + AND NOT EXISTS ( + SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?) + WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`, + [JUNK, JUNK] + ) + round.judged = judged.length + const judgedIds = judged.map((s: { set_id: string }) => s.set_id) + for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH)) + return round + } + + private async drop(setIds: string[]): Promise { + if (setIds.length === 0) return 0 + const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds]) + return result.rowCount ?? 0 + } +} diff --git a/tests/functional/par2_pruner.spec.ts b/tests/functional/par2_pruner.spec.ts new file mode 100644 index 0000000..3d91ebb --- /dev/null +++ b/tests/functional/par2_pruner.spec.ts @@ -0,0 +1,68 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import { Par2Pruner } from '#services/par2_pruner' + +test.group('Par2Pruner', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('drops the file lists of complete and judged sets, keeps the rest', async ({ assert }) => { + const old = new Date(Date.now() - 3 * 86400_000) + let n = 0 + const release = async () => { + n++ + const key = `pp${String(n).padStart(6, '0')}`.padEnd(40, '0') + const [row] = await db + .table('releases') + .insert({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 2, readable: false }) + .returning('id') + return Number(row.id) + } + let candidate = 5000 + const set = async (letter: string, holder: number, files: number, hashedIn: number | null, created = old) => { + const setId = letter.repeat(32) + for (let i = 0; i < files; i++) { + const h = `${letter}${i}`.padEnd(32, '0') + await db.table('par2_files').insert({ set_id: setId, file_id: h, hash16k: h, length: 1000 + i, name: `f${i}.rar` }) + if (hashedIn && i < files) { + candidate++ + await db.table('probe_candidates').insert({ id: candidate, release_id: hashedIn, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' }) + await db.table('file_hashes').insert({ candidate_id: candidate, release_id: hashedIn, hash16k: h, size: 1000 + i }) + } + } + await db.table('par2_sets').insert({ set_id: setId, release_id: holder, files, created_at: created }) + return setId + } + const count = async (setId: string) => Number((await db.from('par2_files').where('set_id', setId).count('* as n'))[0].n) + + // Every file's hash in the holder: complete. + const a = await release() + const complete = await set('a', a, 2, a) + // The files hashed in another release (not merged): kept. + const b = await release() + const apart = await set('b', b, 2, await release()) + // Complete but recent: kept (its merges may come). + const c = await release() + const recent = await set('c', c, 2, c, new Date()) + // Nothing hashed, its only holder dead: judged. + const d = await release() + const dead = await set('d', d, 2, null, new Date()) + await db.table('release_verdicts').insert({ release_id: d, rule: 'dead', rule_version: 1 }) + // Locked isn't junk: kept. + const e = await release() + const locked = await set('e', e, 2, null) + await db.table('release_verdicts').insert({ release_id: e, rule: 'locked', rule_version: 1 }) + + const round = await new Par2Pruner().round() + assert.deepEqual(round, { complete: 1, judged: 1, rows: 4 }) + assert.equal(await count(complete), 0) + assert.equal(await count(apart), 2) + assert.equal(await count(recent), 2) + assert.equal(await count(dead), 0) + assert.equal(await count(locked), 2) + assert.equal((await db.from('par2_sets').count('* as n'))[0].n, '5') + + // Again: the cursor is past them, nothing more. + assert.deepEqual(await new Par2Pruner().round(), { complete: 0, judged: 0, rows: 0 }) + }) +})