Prune par2_files of complete and judged sets (hourly, from the merger)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/par2_merger.ts

@@ -3,6 +3,7 @@ import logger from '@adonisjs/core/services/logger'
 import { par2ReleaseName } from '#lib/release/index'
 import { looksRandom } from '#lib/release/random'
 import { SidecarFolder } from '#services/sidecar_folder'
+import { Par2Pruner } from '#services/par2_pruner'
 
 /** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
 const MAX_SPREAD_MS = 48 * 3600 * 1000
@@ -20,6 +21,8 @@ const ROUND_MS = 60_000
 const OVERLAP_MS = 60_000
 /** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
 const FOLD_EVERY_MS = 5 * 60_000
+/** Between par2_files prunes (complete and judged sets' file lists). */
+const PRUNE_EVERY_MS = 60 * 60_000
 /** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
 const CURSOR = 'merger'
 
@@ -85,9 +88,16 @@ export class Par2Merger {
   }
 
   async run() {
+    let prunedAt = 0
     while (!this.stopped) {
       const started = Date.now()
       try {
+        // First, at start too: a prune needn't wait for a long round.
+        if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
+          prunedAt = Date.now()
+          const pruned = await new Par2Pruner().round()
+          if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
+        }
         const round = await this.round()
         if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
         if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)

app/services/par2_pruner.ts

@@ -0,0 +1,81 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
+const SETTLED_MS = 2 * 86400_000
+/** Sets per statement. */
+const BATCH = 20_000
+/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
+const CURSOR = 'par2-prune'
+/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
+const JUNK = ['dead', 'fake', 'repeated']
+
+export interface PruneRound {
+  complete: number
+  judged: number
+  rows: number
+}
+
+/**
+ * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
+ * 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
+ * done when
+ * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
+ *   set (all of it merged into one);
+ * - judged: every release holding it has a junk verdict (dead, fake, repeated).
+ * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
+ * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
+ */
+export class Par2Pruner {
+  async round(): Promise<PruneRound> {
+    const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
+    const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+    let since: Date = saved ? new Date(saved.at) : new Date(0)
+    const until = new Date(Date.now() - SETTLED_MS)
+    for (;;) {
+      const { rows: sets } = await db.rawQuery(
+        `SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
+         HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
+        [since, until, BATCH]
+      )
+      if (sets.length === 0) break
+      const ids = sets.map((s: { set_id: string }) => s.set_id)
+      const { rows: done } = await db.rawQuery(
+        `SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
+         WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
+           AND NOT EXISTS (
+             SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
+               SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
+               WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
+        [ids]
+      )
+      round.complete += done.length
+      round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
+      since = sets.at(-1).at
+      await db.rawQuery(
+        `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+        [CURSOR, since]
+      )
+      if (sets.length < BATCH) break
+    }
+    // Judged: no cursor (verdicts come and go); only sets with rows left.
+    const { rows: judged } = await db.rawQuery(
+      `SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
+       WHERE v.rule = ANY(?)
+         AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
+         AND NOT EXISTS (
+           SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
+           WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
+      [JUNK, JUNK]
+    )
+    round.judged = judged.length
+    const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
+    for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
+    return round
+  }
+
+  private async drop(setIds: string[]): Promise<number> {
+    if (setIds.length === 0) return 0
+    const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
+    return result.rowCount ?? 0
+  }
+}

tests/functional/par2_pruner.spec.ts

@@ -0,0 +1,68 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { Par2Pruner } from '#services/par2_pruner'
+
+test.group('Par2Pruner', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('drops the file lists of complete and judged sets, keeps the rest', async ({ assert }) => {
+    const old = new Date(Date.now() - 3 * 86400_000)
+    let n = 0
+    const release = async () => {
+      n++
+      const key = `pp${String(n).padStart(6, '0')}`.padEnd(40, '0')
+      const [row] = await db
+        .table('releases')
+        .insert({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 2, readable: false })
+        .returning('id')
+      return Number(row.id)
+    }
+    let candidate = 5000
+    const set = async (letter: string, holder: number, files: number, hashedIn: number | null, created = old) => {
+      const setId = letter.repeat(32)
+      for (let i = 0; i < files; i++) {
+        const h = `${letter}${i}`.padEnd(32, '0')
+        await db.table('par2_files').insert({ set_id: setId, file_id: h, hash16k: h, length: 1000 + i, name: `f${i}.rar` })
+        if (hashedIn && i < files) {
+          candidate++
+          await db.table('probe_candidates').insert({ id: candidate, release_id: hashedIn, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
+          await db.table('file_hashes').insert({ candidate_id: candidate, release_id: hashedIn, hash16k: h, size: 1000 + i })
+        }
+      }
+      await db.table('par2_sets').insert({ set_id: setId, release_id: holder, files, created_at: created })
+      return setId
+    }
+    const count = async (setId: string) => Number((await db.from('par2_files').where('set_id', setId).count('* as n'))[0].n)
+
+    // Every file's hash in the holder: complete.
+    const a = await release()
+    const complete = await set('a', a, 2, a)
+    // The files hashed in another release (not merged): kept.
+    const b = await release()
+    const apart = await set('b', b, 2, await release())
+    // Complete but recent: kept (its merges may come).
+    const c = await release()
+    const recent = await set('c', c, 2, c, new Date())
+    // Nothing hashed, its only holder dead: judged.
+    const d = await release()
+    const dead = await set('d', d, 2, null, new Date())
+    await db.table('release_verdicts').insert({ release_id: d, rule: 'dead', rule_version: 1 })
+    // Locked isn't junk: kept.
+    const e = await release()
+    const locked = await set('e', e, 2, null)
+    await db.table('release_verdicts').insert({ release_id: e, rule: 'locked', rule_version: 1 })
+
+    const round = await new Par2Pruner().round()
+    assert.deepEqual(round, { complete: 1, judged: 1, rows: 4 })
+    assert.equal(await count(complete), 0)
+    assert.equal(await count(apart), 2)
+    assert.equal(await count(recent), 2)
+    assert.equal(await count(dead), 0)
+    assert.equal(await count(locked), 2)
+    assert.equal((await db.from('par2_sets').count('* as n'))[0].n, '5')
+
+    // Again: the cursor is past them, nothing more.
+    assert.deepEqual(await new Par2Pruner().round(), { complete: 0, judged: 0, rows: 0 })
+  })
+})