The par2 merge: releases whose files one par2 set describes become one

The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.

Also: corpus_sweep.py counts the marked ranges without psql's command tag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/par2_merger.ts

@@ -0,0 +1,154 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { par2ReleaseName } from '#lib/release/index'
+import { looksRandom } from '#lib/release/random'
+
+/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+/** Between rounds. */
+const ROUND_MS = 60_000
+/** A round looks back this far past the previous one's start (rows committed while it ran). */
+const OVERLAP_MS = 60_000
+
+export interface MergeRound {
+  sets: number
+  merges: number
+  releases: number
+}
+
+interface Member {
+  id: number
+  release_key: string
+  group_id: number
+  files: number
+  posted_at: Date | null
+  readable: boolean
+}
+
+/**
+ * The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
+ * describes are one upload. Obfuscated uploads post each file under a name of
+ * its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
+ * one-file releases), so the header pass can't group them; the prober reads
+ * each such file's first article (a `lone` candidate) and stores the MD5 of
+ * its first 16 KiB, and a par2 body's file list holds the same hash per file,
+ * whatever the file was renamed to. Releases in one group, posted within
+ * MAX_SPREAD_MS of a release that holds the set's par2, become the one with
+ * the most files: its chunks, probe candidates, inspections and hashes, the
+ * sums of its counts, and when none of them has a readable name the name the
+ * par2's file names share. Each merge is kept in release_merges.
+ */
+export class Par2Merger {
+  private stopped = false
+  private since = new Date(0)
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        const round = await this.round()
+        if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
+      } catch (error: any) {
+        logger.error(`Par2 merge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
+  async round(): Promise<MergeRound> {
+    const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
+    const since = new Date(this.since.getTime() - OVERLAP_MS)
+    const { rows } = await db.rawQuery(
+      `SELECT set_id FROM par2_sets WHERE created_at >= ?
+       UNION
+       SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
+      [since, since]
+    )
+    const round: MergeRound = { sets: rows.length, merges: 0, releases: 0 }
+    for (const { set_id: setId } of rows) {
+      if (this.stopped) return round
+      const merged = await this.mergeSet(setId.trim())
+      if (merged > 0) {
+        round.merges++
+        round.releases += merged + 1
+      }
+    }
+    this.since = now[0].at
+    return round
+  }
+
+  /** Merges the releases of one set; how many went into another. */
+  async mergeSet(setId: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows } = await trx.rawQuery(
+        `WITH m AS (
+           SELECT release_id, true AS holds FROM par2_sets WHERE set_id = ?
+           UNION ALL
+           SELECT h.release_id, false FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length WHERE f.set_id = ?
+         ), d AS (SELECT release_id, bool_or(holds) AS holds FROM m GROUP BY 1)
+         SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, d.holds
+         FROM releases r JOIN d ON d.release_id = r.id ORDER BY r.id FOR UPDATE OF r`,
+        [setId, setId]
+      )
+      const anchor = rows.find((r: Member & { holds: boolean }) => r.holds)
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const near = (r: Member) => at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS
+      const members: Member[] = rows.filter((r: Member) => r.group_id === anchor.group_id && near(r))
+      if (members.length < 2) return 0
+      const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
+      const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
+      const name = members.every((m) => !m.readable) ? await this.setName(trx, setId) : undefined
+
+      await trx.rawQuery(
+        `INSERT INTO release_merges (from_id, from_key, into_id, set_id, files)
+         SELECT id, release_key, ?, ?, files FROM releases WHERE id = ANY(?)`,
+        [target.id, setId, others]
+      )
+      for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
+        await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      }
+      // A set held by the target already isn't held twice.
+      await trx.rawQuery(`DELETE FROM par2_sets WHERE release_id = ANY(?) AND set_id IN (SELECT set_id FROM par2_sets WHERE release_id = ?)`, [others, target.id])
+      await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      await trx.rawQuery(
+        `UPDATE releases t SET
+           files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
+           first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
+           posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
+           first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
+           readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
+           file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
+           name = coalesce(nullif(?, ''), t.name)
+         FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
+                 min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
+                 max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
+                 bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
+                 max(file_total) AS file_total, max(inner_name) AS inner_name
+               FROM releases WHERE id = ANY(?)) s
+         WHERE t.id = ?`,
+        [name ?? '', others, target.id]
+      )
+      // Completeness as the settler decides it, once settled.
+      await trx.rawQuery(
+        `UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
+        [target.id]
+      )
+      await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
+      return others.length
+    })
+  }
+
+  /** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
+  private async setName(trx: any, setId: string): Promise<string | undefined> {
+    const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
+    const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
+    // A par2 made after the renaming names the random names again.
+    return name && !looksRandom(name) ? name.slice(0, 255) : undefined
+  }
+}

app/services/prober.ts

@@ -159,8 +159,10 @@ export class Prober {
         `INSERT INTO release_inspections (release_id, candidate_id, kind, result)
          SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
          ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
-        [JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: o.inspection })))]
+        // A par2 set's file list goes to par2_files, not into every inspection.
+        [JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
       )
+      await this.storeForMerge(trx, done)
       // Each touched release from all its inspections (earlier rounds may have stored others).
       const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
       const { rows } = await trx.rawQuery(
@@ -187,6 +189,30 @@ export class Prober {
     })
   }
 
+  /** What the par2 merge matches (app/services/par2_merger.ts): files' first-16-KiB hashes, par2 sets' file lists. */
+  private async storeForMerge(trx: any, done: Outcome[]) {
+    const hashed = done.filter((o) => o.inspection!.hash16k)
+    if (hashed.length > 0) {
+      await trx.rawQuery(
+        `INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
+         SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
+         ON CONFLICT (candidate_id) DO NOTHING`,
+        [JSON.stringify(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
+      )
+    }
+    for (const o of done) {
+      const set = o.inspection!.par2
+      if (!set) continue
+      await trx.rawQuery(
+        `INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
+         SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
+         ON CONFLICT DO NOTHING`,
+        [set.setId, JSON.stringify(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
+      )
+      await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
+    }
+  }
+
   private report() {
     const seconds = (Date.now() - this.stats.since) / 1000
     if (seconds < REPORT_MS / 1000) return