Release completeness check without reading every binary; SEED_RANGES
release_complete() matched each par2-listed file through a correlated subquery that scanned all binaries per listed file, run twice per inserted release_files row: one insert of a few thousand rows ran for 20+ minutes. It now reads the release's files once by release id, and release_files refreshes once per INSERT statement. seed:node's SEED_CONNECTIONS/--connections is now SEED_RANGES/--ranges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2dbca9b152
commit
4d17c5a8e8
5 files changed
+135
-11
No files matched your search
@@ -0,0 +1,69 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Release completeness (D24) without cubic cost (LOG.md 2026-10-06 (4)). A
|
||||
* par2 set listing n files was inserted into release_files row by row, each
|
||||
* row's trigger ran release_complete() twice, and that matched every listed
|
||||
* file against all binaries in the database: n × 2 × n × (all binaries). One
|
||||
* insert of a few thousand rows ran for over 20 minutes at full CPU.
|
||||
*
|
||||
* - release_complete(): the release's files are read once, by release id,
|
||||
* and each listed file is matched against those by size or by name. The
|
||||
* correlated subquery read every binary in the database per listed file
|
||||
* (binaries_size_index), filtering by release only afterwards.
|
||||
* - refresh_release_complete(): computes it once.
|
||||
* - release_files: one refresh per release per INSERT statement.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||
WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
|
||||
-- The release's files, once, by index (read only when there's a par2 listing to check).
|
||||
have AS MATERIALIZED (
|
||||
SELECT b.size, f.real_name FROM files f JOIN binaries b ON b.file_id = f.id WHERE f.release_id = rid)
|
||||
SELECT coalesce((
|
||||
SELECT CASE
|
||||
WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
|
||||
WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
|
||||
SELECT 1 FROM release_files rf
|
||||
WHERE rf.release_id = rid AND rf.size >= 65536
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
|
||||
AND NOT EXISTS (SELECT 1 FROM have h WHERE h.real_name = rf.name))
|
||||
ELSE false END
|
||||
FROM r), false)
|
||||
$$`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases r SET complete = c.complete
|
||||
FROM (SELECT release_complete(rid) AS complete) c
|
||||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||||
$$`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE OR REPLACE FUNCTION release_files_release_complete_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM refresh_release_complete(n.release_id) FROM (SELECT DISTINCT release_id FROM new_rows) n;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
// Swapping the trigger locks release_files: inserts still running the old cubic check
|
||||
// (minutes) are cancelled rather than waited for; their jobs are retried, and run fast.
|
||||
await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
|
||||
await this.db.rawQuery(`
|
||||
SELECT pg_cancel_backend(pid) FROM pg_stat_activity
|
||||
WHERE pid <> pg_backend_pid() AND state = 'active' AND query ILIKE 'insert into "release_files"%'
|
||||
AND now() - query_start > interval '30 seconds'`)
|
||||
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
|
||||
REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION release_files_release_complete_statement()`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
await this.db.rawQuery(`DROP TRIGGER IF EXISTS release_files_release_complete ON release_files`)
|
||||
await this.db.rawQuery(`
|
||||
CREATE TRIGGER release_files_release_complete AFTER INSERT ON release_files
|
||||
FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`)
|
||||
await this.db.rawQuery(`DROP FUNCTION IF EXISTS release_files_release_complete_statement()`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user