usenet-indexer/app/services/par2_pruner.ts

import db from '@adonisjs/lucid/services/db'
import { decidePar2Completeness } from '#services/par2_completeness'

/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
const SETTLED_MS = 2 * 86400_000
/** Sets per statement. */
const BATCH = 20_000
/** The sweep's row in worker_cursors: the last set_id it looked at (key); past the last set it starts over. */
const CURSOR = 'par2-prune'
/** The merger's row in worker_cursors: the end of its last whole window. */
const MERGER = 'merger'
/** The merger's windows look back this far (Par2Merger's OVERLAP_MS). */
const MERGER_OVERLAP_MS = 60_000
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
const JUNK = ['dead', 'fake', 'repeated']
/** Probe candidates whose results a merge reads: a file's hash (lone), a set's file list (par2). */
const MERGE_KINDS = ['lone', 'par2']

export interface PruneRound {
  complete: number
  judged: number
  final: number
  rows: number
}

/**
 * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60), 2026-10-20 (87)). A set's file list
 * matches file hashes to releases and names an unreadable upload once; it's done when
 * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
 *   set (all of it merged into one);
 * - judged: every release holding it has a junk verdict (dead, fake, repeated);
 * - final: SETTLED_MS after its last holder's par2, every holder is final (release_final: a run of done ranges holds
 *   it with hours to spare, so no more of its upload will be read), the candidates of the ranges around it are probed,
 *   and the merger has seen every hash of its files. A file still missing then is one no probe will hash: most sets
 *   describe a readable upload's files, which aren't lone candidates (9 in 10 sets were kept for that, 19 GB).
 * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
 * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
 * The sweep goes through the sets with rows by set_id and starts over at the end: a set kept now (a holder at the edge
 * of a run, its ranges' probes not done yet) is looked at again on the next pass.
 */
export class Par2Pruner {
  /** Sets per batch (BATCH; smaller in the tests). */
  constructor(private batch = BATCH) {}

  /**
   * Batches until a pass is done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from
   * it): run from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md
   * 2026-10-17 (72)).
   */
  async round(budgetMs = Infinity): Promise<PruneRound> {
    const deadline = Date.now() + budgetMs
    const round: PruneRound = { complete: 0, judged: 0, final: 0, rows: 0 }
    const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
    let after: string = cursors[0]?.key ?? ''
    // Ranges done since the oldest merge-kind candidate still unprobed: their releases' hashes and file lists may be
    // still to come. Hashes since the merger's last window: it hasn't matched them yet.
    const { rows: marks } = await db.rawQuery(
      `SELECT coalesce((SELECT min(created_at) FROM probe_candidates WHERE status IN ('waiting', 'taken') AND kind = ANY(?)), now()) AS probed,
              coalesce((SELECT at FROM worker_cursors WHERE name = ?), to_timestamp(0)) - make_interval(secs => ?) AS merged,
              now() - make_interval(secs => ?) AS settled`,
      [MERGE_KINDS, MERGER, MERGER_OVERLAP_MS / 1000, SETTLED_MS / 1000]
    )
    const { probed, merged, settled } = marks[0]
    for (;;) {
      const { rows: sets } = await db.rawQuery(
        `WITH batch AS MATERIALIZED (SELECT DISTINCT set_id FROM par2_files WHERE set_id > ? ORDER BY set_id LIMIT ?),
         recent AS MATERIALIZED (SELECT group_id, posted_low, posted_high FROM ranges WHERE status = 'done' AND done_at >= ?),
         unseen AS MATERIALIZED (SELECT DISTINCT hash16k, size FROM file_hashes WHERE created_at >= ?)
         SELECT b.set_id, CASE
           WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
            AND NOT EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = b.set_id AND NOT EXISTS (
                  SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
                  WHERE h.hash16k = f.hash16k AND h.size = f.length)) THEN 'complete'
           WHEN NOT EXISTS (SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts v ON v.release_id = o.release_id AND v.rule = ANY(?)
                  WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
           WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
            AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
                  release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) IS NOT TRUE
                  OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
                    AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
            AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
           THEN 'final' END AS reason
         FROM batch b ORDER BY b.set_id`,
        [after, this.batch, probed, merged, settled, JUNK, settled]
      )
      const done = (reason: keyof PruneRound) => sets.filter((s: { reason: string }) => s.reason === reason).map((s: { set_id: string }) => s.set_id)
      for (const reason of ['complete', 'judged', 'final'] as const) {
        const ids = done(reason)
        round[reason] += ids.length
        // The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
        if (reason !== 'judged' && ids.length > 0) {
          const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
          await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
        }
        round.rows += await this.drop(ids)
      }
      // Past the last set: the next batch starts the next pass.
      after = sets.length < this.batch ? '' : sets.at(-1).set_id
      await db.rawQuery(
        `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
        [CURSOR, after]
      )
      if (after === '' || Date.now() > deadline) break
    }
    return round
  }

  private async drop(setIds: string[]): Promise<number> {
    if (setIds.length === 0) return 0
    const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
    return result.rowCount ?? 0
  }
}