Release completeness check without reading every binary; SEED_RANGES
Test and deploy / test (push) Successful in 2m15s
Test and deploy / deploy (push) Successful in 2m26s

release_complete() matched each par2-listed file through a correlated
subquery that scanned all binaries per listed file, run twice per
inserted release_files row: one insert of a few thousand rows ran for
20+ minutes. It now reads the release's files once by release id, and
release_files refreshes once per INSERT statement.

seed:node's SEED_CONNECTIONS/--connections is now SEED_RANGES/--ranges.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 12:10:57 +02:00
1 parent 2dbca9b152
commit 4d17c5a8e8
5 files changed
+135 -11

No files matched your search

@@ -0,0 +1,69 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* Release completeness (D24) without cubic cost (LOG.md 2026-10-06 (4)). A
* par2 set listing n files was inserted into release_files row by row, each
* row's trigger ran release_complete() twice, and that matched every listed
* file against all binaries in the database: n × 2 × n × (all binaries). One
* insert of a few thousand rows ran for over 20 minutes at full CPU.
*
* - release_complete(): the release's files are read once, by release id,
* and each listed file is matched against those by size or by name. The
* correlated subquery read every binary in the database per listed file
* (binaries_size_index), filtering by release only afterwards.
* - refresh_release_complete(): computes it once.
* - release_files: one refresh per release per INSERT statement.
*/
export default class extends BaseSchema {
async up() {
await this.db.rawQuery(`
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
-- The release's files, once, by index (read only when there's a par2 listing to check).
have AS MATERIALIZED (
SELECT b.size, f.real_name FROM files f JOIN binaries b ON b.file_id = f.id WHERE f.release_id = rid)
SELECT coalesce((
SELECT CASE
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
SELECT 1 FROM release_files rf
WHERE rf.release_id = rid AND rf.size >= 65536
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.real_name = rf.name))
ELSE false END
FROM r), false)
$$`)
await this.db.rawQuery(`
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
UPDATE releases r SET complete = c.complete
FROM (SELECT release_complete(rid) AS complete) c
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
$$`)
await this.db.rawQuery(`
CREATE OR REPLACE FUNCTION release_files_release_complete_statement() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
PERFORM refresh_release_complete(n.release_id) FROM (SELECT DISTINCT release_id FROM new_rows) n;
RETURN NULL;
END $$`)
// Swapping the trigger locks release_files: inserts still running the old cubic check
// (minutes) are cancelled rather than waited for; their jobs are retried, and run fast.
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
await this.db.rawQuery(`
SELECT pg_cancel_backend(pid) FROM pg_stat_activity
WHERE pid <> pg_backend_pid() AND state = 'active' AND query ILIKE 'insert into "release_files"%'
AND now() - query_start > interval '30 seconds'`)
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
await this.db.rawQuery(`
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT
EXECUTE FUNCTION release_files_release_complete_statement()`)
}
async down() {
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
await this.db.rawQuery(`
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
await this.db.rawQuery(`DROP FUNCTION IF EXISTS release_files_release_complete_statement()`)
}
}