Daan MeijerandClaude Opus 5.5
4d17c5a8e8
release_complete() matched each par2-listed file through a correlated subquery that scanned all binaries per listed file, run twice per inserted release_files row: one insert of a few thousand rows ran for 20+ minutes. It now reads the release's files once by release id, and release_files refreshes once per INSERT statement. seed:node's SEED_CONNECTIONS/--connections is now SEED_RANGES/--ranges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
70 lines
3.7 KiB
TypeScript
70 lines
3.7 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||
|
||
/**
|
||
* Release completeness (D24) without cubic cost (LOG.md 2026-10-06 (4)). A
|
||
* par2 set listing n files was inserted into release_files row by row, each
|
||
* row's trigger ran release_complete() twice, and that matched every listed
|
||
* file against all binaries in the database: n × 2 × n × (all binaries). One
|
||
* insert of a few thousand rows ran for over 20 minutes at full CPU.
|
||
*
|
||
* - release_complete(): the release's files are read once, by release id,
|
||
* and each listed file is matched against those by size or by name. The
|
||
* correlated subquery read every binary in the database per listed file
|
||
* (binaries_size_index), filtering by release only afterwards.
|
||
* - refresh_release_complete(): computes it once.
|
||
* - release_files: one refresh per release per INSERT statement.
|
||
*/
|
||
export default class extends BaseSchema {
|
||
async up() {
|
||
await this.db.rawQuery(`
|
||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||
WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
|
||
-- The release's files, once, by index (read only when there's a par2 listing to check).
|
||
have AS MATERIALIZED (
|
||
SELECT b.size, f.real_name FROM files f JOIN binaries b ON b.file_id = f.id WHERE f.release_id = rid)
|
||
SELECT coalesce((
|
||
SELECT CASE
|
||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
|
||
SELECT 1 FROM release_files rf
|
||
WHERE rf.release_id = rid AND rf.size >= 65536
|
||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
|
||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.real_name = rf.name))
|
||
ELSE false END
|
||
FROM r), false)
|
||
$$`)
|
||
await this.db.rawQuery(`
|
||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||
UPDATE releases r SET complete = c.complete
|
||
FROM (SELECT release_complete(rid) AS complete) c
|
||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||
$$`)
|
||
await this.db.rawQuery(`
|
||
CREATE OR REPLACE FUNCTION release_files_release_complete_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
PERFORM refresh_release_complete(n.release_id) FROM (SELECT DISTINCT release_id FROM new_rows) n;
|
||
RETURN NULL;
|
||
END $$`)
|
||
// Swapping the trigger locks release_files: inserts still running the old cubic check
|
||
// (minutes) are cancelled rather than waited for; their jobs are retried, and run fast.
|
||
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
|
||
await this.db.rawQuery(`
|
||
SELECT pg_cancel_backend(pid) FROM pg_stat_activity
|
||
WHERE pid <> pg_backend_pid() AND state = 'active' AND query ILIKE 'insert into "release_files"%'
|
||
AND now() - query_start > interval '30 seconds'`)
|
||
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
|
||
await this.db.rawQuery(`
|
||
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
|
||
REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT
|
||
EXECUTE FUNCTION release_files_release_complete_statement()`)
|
||
}
|
||
|
||
async down() {
|
||
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
|
||
await this.db.rawQuery(`
|
||
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
|
||
FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
|
||
await this.db.rawQuery(`DROP FUNCTION IF EXISTS release_files_release_complete_statement()`)
|
||
}
|
||
}
|