Judge: count a chunk's file hashes once, from a covering index

The incremental judge counted each changed release's files twice, each
index entry a heap fetch: 24-32 s a chunk. file_hashes by (hash16k,
size, release_id) replaces (hash16k); the other releases of a file are
judged only when the chunk brings it to 20.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/release_judge.ts

@@ -39,10 +39,7 @@ export interface Rule {
 /** A join to the releases in scope (an incremental round), on the given release id column. */
 const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
 
-/** The file hashes (first 16 KiB and size) of the releases in scope that are in REPEATED_RELEASES+ releases. */
+
-const TIPPED = `hashes AS (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id),
-  tipped AS (SELECT x.hash16k, x.size FROM hashes x
-    WHERE (SELECT count(DISTINCT f.release_id) FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) >= ${REPEATED_RELEASES})`
 
 /**
  * The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
@@ -73,11 +70,11 @@ export const RULES: Rule[] = [
   {
     name: 'repeated',
     says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
-    // Scoped: the hashes of the releases in scope, counted each (a round first adds the other releases a hash of theirs
+    // Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
-    // tipped over: ReleaseJudge.changed).
+    // (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
     sql: (scoped) =>
       scoped
-        ? `WITH ${TIPPED} SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN tipped USING (hash16k, size)`
+        ? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
         : `SELECT DISTINCT h.release_id FROM file_hashes h
           JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
   },
@@ -191,7 +188,8 @@ export class ReleaseJudge {
   /**
    * The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
    * A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
-   * already repeated, or dead or fake, which come first).
+   * already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
+   * file in that many without them had its releases judged before.
    */
   async changed(): Promise<JudgeRound> {
     const round: JudgeRound = { releases: 0, rules: {} }
@@ -211,10 +209,24 @@ export class ReleaseJudge {
         const taken = Number(rows[0].taken)
         more = taken === CHUNK
         if (taken === 0) return
+        // The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
+        await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
         await trx.rawQuery(
-          `WITH ${TIPPED}
+          `CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
-           INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN tipped USING (hash16k, size)
+        )
-           WHERE NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
+        await trx.rawQuery(
+          `INSERT INTO judge_tipped
+           SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
+           FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
+           CROSS JOIN LATERAL (
+             SELECT count(DISTINCT f.release_id) AS total,
+               count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
+             FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
+           WHERE c.total >= ${REPEATED_RELEASES}`
+        )
+        await trx.rawQuery(
+          `INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
+           WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
            ON CONFLICT DO NOTHING`
         )
         await trx.rawQuery(`ANALYZE judge_scope`)

database/migrations/1770000005000_file_hashes_hash_size_release.ts

@@ -0,0 +1,21 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * file_hashes by (hash16k, size, release_id), in place of (hash16k): how many releases have a file, from the index
+ * alone. The judge counts that for every file of each changed release (repeated); with the row fetched for size and
+ * release_id per entry, a file posted in 100k releases cost 100k heap reads every round (24–32 s a chunk; LOG.md
+ * 2026-10-19 (84)). The merger and pruner join on (hash16k, size) too.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_size_release_idx ON file_hashes (hash16k, size, release_id)`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_idx`)
+  }
+
+  async down() {
+    this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_idx ON file_hashes (hash16k)`)
+    this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_size_release_idx`)
+  }
+}

tests/functional/release_judge.spec.ts

@@ -171,6 +171,12 @@ test.group('ReleaseJudge', (group) => {
     const rules = new Set()
     for (const id of [...holders, twentieth]) rules.add(await verdict(id))
     assert.deepEqual([...rules], ['repeated'])
+    // A 21st: the file was in 20 releases without it, so only it is judged.
+    const twentyFirst = await release('holder20')
+    await db.table('file_hashes').insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 })
+    round = await judge.changed()
+    assert.equal(round.releases, 1)
+    assert.equal(await verdict(twentyFirst), 'repeated')
 
     // A verdict nothing noted stays until the full round; it's due a day after the last one.
     const stale = await release('Fine.Release')