Prune par2_files of complete and judged sets (hourly, from the merger)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/par2_merger.ts

@@ -3,6 +3,7 @@ import logger from '@adonisjs/core/services/logger'
 import { par2ReleaseName } from '#lib/release/index'
 import { looksRandom } from '#lib/release/random'
 import { SidecarFolder } from '#services/sidecar_folder'
+import { Par2Pruner } from '#services/par2_pruner'
 
 /** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
 const MAX_SPREAD_MS = 48 * 3600 * 1000
@@ -20,6 +21,8 @@ const ROUND_MS = 60_000
 const OVERLAP_MS = 60_000
 /** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
 const FOLD_EVERY_MS = 5 * 60_000
+/** Between par2_files prunes (complete and judged sets' file lists). */
+const PRUNE_EVERY_MS = 60 * 60_000
 /** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
 const CURSOR = 'merger'
 
@@ -85,9 +88,16 @@ export class Par2Merger {
   }
 
   async run() {
+    let prunedAt = 0
     while (!this.stopped) {
       const started = Date.now()
       try {
+        // First, at start too: a prune needn't wait for a long round.
+        if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
+          prunedAt = Date.now()
+          const pruned = await new Par2Pruner().round()
+          if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
+        }
         const round = await this.round()
         if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
         if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)

app/services/par2_pruner.ts

@@ -0,0 +1,81 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
+const SETTLED_MS = 2 * 86400_000
+/** Sets per statement. */
+const BATCH = 20_000
+/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
+const CURSOR = 'par2-prune'
+/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
+const JUNK = ['dead', 'fake', 'repeated']
+
+export interface PruneRound {
+  complete: number
+  judged: number
+  rows: number
+}
+
+/**
+ * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
+ * 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
+ * done when
+ * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
+ *   set (all of it merged into one);
+ * - judged: every release holding it has a junk verdict (dead, fake, repeated).
+ * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
+ * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
+ */
+export class Par2Pruner {
+  async round(): Promise<PruneRound> {
+    const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
+    const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+    let since: Date = saved ? new Date(saved.at) : new Date(0)
+    const until = new Date(Date.now() - SETTLED_MS)
+    for (;;) {
+      const { rows: sets } = await db.rawQuery(
+        `SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
+         HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
+        [since, until, BATCH]
+      )
+      if (sets.length === 0) break
+      const ids = sets.map((s: { set_id: string }) => s.set_id)
+      const { rows: done } = await db.rawQuery(
+        `SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
+         WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
+           AND NOT EXISTS (
+             SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
+               SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
+               WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
+        [ids]
+      )
+      round.complete += done.length
+      round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
+      since = sets.at(-1).at
+      await db.rawQuery(
+        `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+        [CURSOR, since]
+      )
+      if (sets.length < BATCH) break
+    }
+    // Judged: no cursor (verdicts come and go); only sets with rows left.
+    const { rows: judged } = await db.rawQuery(
+      `SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
+       WHERE v.rule = ANY(?)
+         AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
+         AND NOT EXISTS (
+           SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
+           WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
+      [JUNK, JUNK]
+    )
+    round.judged = judged.length
+    const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
+    for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
+    return round
+  }
+
+  private async drop(setIds: string[]): Promise<number> {
+    if (setIds.length === 0) return 0
+    const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
+    return result.rowCount ?? 0
+  }
+}