Judge: count a chunk's file hashes once, from a covering index

The incremental judge counted each changed release's files twice, each
index entry a heap fetch: 24-32 s a chunk. file_hashes by (hash16k,
size, release_id) replaces (hash16k); the other releases of a file are
judged only when the chunk brings it to 20.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/release_judge.ts

@@ -39,10 +39,7 @@ export interface Rule {
 /** A join to the releases in scope (an incremental round), on the given release id column. */
 const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
 
-/** The file hashes (first 16 KiB and size) of the releases in scope that are in REPEATED_RELEASES+ releases. */
-const TIPPED = `hashes AS (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id),
-  tipped AS (SELECT x.hash16k, x.size FROM hashes x
-    WHERE (SELECT count(DISTINCT f.release_id) FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) >= ${REPEATED_RELEASES})`
+
 
 /**
  * The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
@@ -73,11 +70,11 @@ export const RULES: Rule[] = [
   {
     name: 'repeated',
     says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
-    // Scoped: the hashes of the releases in scope, counted each (a round first adds the other releases a hash of theirs
-    // tipped over: ReleaseJudge.changed).
+    // Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
+    // (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
     sql: (scoped) =>
       scoped
-        ? `WITH ${TIPPED} SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN tipped USING (hash16k, size)`
+        ? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
         : `SELECT DISTINCT h.release_id FROM file_hashes h
           JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
   },
@@ -191,7 +188,8 @@ export class ReleaseJudge {
   /**
    * The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
    * A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
-   * already repeated, or dead or fake, which come first).
+   * already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
+   * file in that many without them had its releases judged before.
    */
   async changed(): Promise<JudgeRound> {
     const round: JudgeRound = { releases: 0, rules: {} }
@@ -211,10 +209,24 @@ export class ReleaseJudge {
         const taken = Number(rows[0].taken)
         more = taken === CHUNK
         if (taken === 0) return
+        // The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
+        await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
         await trx.rawQuery(
-          `WITH ${TIPPED}
-           INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN tipped USING (hash16k, size)
-           WHERE NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
+          `CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
+        )
+        await trx.rawQuery(
+          `INSERT INTO judge_tipped
+           SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
+           FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
+           CROSS JOIN LATERAL (
+             SELECT count(DISTINCT f.release_id) AS total,
+               count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
+             FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
+           WHERE c.total >= ${REPEATED_RELEASES}`
+        )
+        await trx.rawQuery(
+          `INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
+           WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
            ON CONFLICT DO NOTHING`
         )
         await trx.rawQuery(`ANALYZE judge_scope`)