From 9725bdbd7b10469e1e0734dbd4e56a1befdf0d8b Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 19 Oct 2026 20:32:52 +0200 Subject: [PATCH] Judge: count a chunk's file hashes once, from a covering index The incremental judge counted each changed release's files twice, each index entry a heap fetch: 24-32 s a chunk. file_hashes by (hash16k, size, release_id) replaces (hash16k); the other releases of a file are judged only when the chunk brings it to 20. Co-Authored-By: Claude Opus 5.5 --- app/services/release_judge.ts | 34 +++++++++++++------ ...000005000_file_hashes_hash_size_release.ts | 21 ++++++++++++ tests/functional/release_judge.spec.ts | 6 ++++ 3 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 database/migrations/1770000005000_file_hashes_hash_size_release.ts diff --git a/app/services/release_judge.ts b/app/services/release_judge.ts index c2b70b8..9f455c9 100644 --- a/app/services/release_judge.ts +++ b/app/services/release_judge.ts @@ -39,10 +39,7 @@ export interface Rule { /** A join to the releases in scope (an incremental round), on the given release id column. */ const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '') -/** The file hashes (first 16 KiB and size) of the releases in scope that are in REPEATED_RELEASES+ releases. */ -const TIPPED = `hashes AS (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id), - tipped AS (SELECT x.hash16k, x.size FROM hashes x - WHERE (SELECT count(DISTINCT f.release_id) FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) >= ${REPEATED_RELEASES})` + /** * The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what @@ -73,11 +70,11 @@ export const RULES: Rule[] = [ { name: 'repeated', says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`, - // Scoped: the hashes of the releases in scope, counted each (a round first adds the other releases a hash of theirs - // tipped over: ReleaseJudge.changed). + // Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk + // (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over). sql: (scoped) => scoped - ? `WITH ${TIPPED} SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN tipped USING (hash16k, size)` + ? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)` : `SELECT DISTINCT h.release_id FROM file_hashes h JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`, }, @@ -191,7 +188,8 @@ export class ReleaseJudge { /** * The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted). * A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those - * already repeated, or dead or fake, which come first). + * already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a + * file in that many without them had its releases judged before. */ async changed(): Promise { const round: JudgeRound = { releases: 0, rules: {} } @@ -211,10 +209,24 @@ export class ReleaseJudge { const taken = Number(rows[0].taken) more = taken === CHUNK if (taken === 0) return + // The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000). + await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`) await trx.rawQuery( - `WITH ${TIPPED} - INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN tipped USING (hash16k, size) - WHERE NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated')) + `CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP` + ) + await trx.rawQuery( + `INSERT INTO judge_tipped + SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES} + FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x + CROSS JOIN LATERAL ( + SELECT count(DISTINCT f.release_id) AS total, + count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside + FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c + WHERE c.total >= ${REPEATED_RELEASES}` + ) + await trx.rawQuery( + `INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size) + WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated')) ON CONFLICT DO NOTHING` ) await trx.rawQuery(`ANALYZE judge_scope`) diff --git a/database/migrations/1770000005000_file_hashes_hash_size_release.ts b/database/migrations/1770000005000_file_hashes_hash_size_release.ts new file mode 100644 index 0000000..3d19196 --- /dev/null +++ b/database/migrations/1770000005000_file_hashes_hash_size_release.ts @@ -0,0 +1,21 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * file_hashes by (hash16k, size, release_id), in place of (hash16k): how many releases have a file, from the index + * alone. The judge counts that for every file of each changed release (repeated); with the row fetched for size and + * release_id per entry, a file posted in 100k releases cost 100k heap reads every round (24–32 s a chunk; LOG.md + * 2026-10-19 (84)). The merger and pruner join on (hash16k, size) too. + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_size_release_idx ON file_hashes (hash16k, size, release_id)`) + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_idx`) + } + + async down() { + this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS file_hashes_hash_idx ON file_hashes (hash16k)`) + this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS file_hashes_hash_size_release_idx`) + } +} diff --git a/tests/functional/release_judge.spec.ts b/tests/functional/release_judge.spec.ts index a166422..de02bd1 100644 --- a/tests/functional/release_judge.spec.ts +++ b/tests/functional/release_judge.spec.ts @@ -171,6 +171,12 @@ test.group('ReleaseJudge', (group) => { const rules = new Set() for (const id of [...holders, twentieth]) rules.add(await verdict(id)) assert.deepEqual([...rules], ['repeated']) + // A 21st: the file was in 20 releases without it, so only it is judged. + const twentyFirst = await release('holder20') + await db.table('file_hashes').insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 }) + round = await judge.changed() + assert.equal(round.releases, 1) + assert.equal(await verdict(twentyFirst), 'repeated') // A verdict nothing noted stays until the full round; it's due a day after the last one. const stale = await release('Fine.Release')