Par2 size matching 100x faster; release bookkeeping once per statement
matchPar2Releases checked every same-size release_files row's set and date (18 s per batch of body-derived binaries); now an index on release_files (release_id, size) drops the repeated sizes first: 0.16 s. File inserts updated their release in four row triggers; one statement trigger now updates each release once per INSERT. Completeness is noted in release_dirty and recomputed by gc:scheduler every 30 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
288a6bab8d
commit
be2112f4ad
9 files changed
+208
-11
No files matched your search
@@ -0,0 +1,107 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Cheaper release bookkeeping for file inserts (LOG.md 2026-10-06 (6)).
|
||||
*
|
||||
* Every inserted file fired four row triggers, each updating its release:
|
||||
* file_count/size, search_files, extensions and completeness, so a seed
|
||||
* batch of thousands of files rewrote its releases thousands of times (a
|
||||
* quarter of Postgres' active time on homelab). Now one statement-level
|
||||
* trigger updates each release once per INSERT, with all four.
|
||||
*
|
||||
* Completeness (D24), the expensive part, is no longer computed in triggers:
|
||||
* refresh_release_complete() only notes the release in release_dirty, and
|
||||
* gc:scheduler recomputes the noted releases every 30 minutes
|
||||
* (recompute_release_complete). Counts, sizes, search text and extensions
|
||||
* stay current: code decides on them right away (weak releases, deleting
|
||||
* empty releases).
|
||||
*
|
||||
* Also an index on release_files (release_id, size) for the par2 size
|
||||
* match's "listed once in its set" check (IngestService.matchPar2Releases).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
// Swapping triggers locks files and its partitions: give up rather than queue every writer behind it.
|
||||
await q(`SET LOCAL lock_timeout = '2min'`)
|
||||
// Built CONCURRENTLY by hand on homelab (2026-10-06); IF NOT EXISTS keeps that one.
|
||||
await q(`CREATE INDEX IF NOT EXISTS release_files_release_size_idx ON release_files (release_id, size)`)
|
||||
|
||||
// Append-only (no key): writers never wait on each other or on the sweep.
|
||||
await q(`CREATE TABLE IF NOT EXISTS release_dirty (release_id bigint NOT NULL)`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION recompute_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases r SET complete = c.complete
|
||||
FROM (SELECT release_complete(rid) AS complete) c
|
||||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||||
$$`)
|
||||
// Every trigger that called this (files, release_files, binaries.size, releases.file_total) now just notes the release.
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL
|
||||
$$`)
|
||||
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
WITH added AS (
|
||||
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
||||
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
||||
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
||||
FROM (
|
||||
SELECT release_id, size, name_extension(nm) AS e,
|
||||
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
||||
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
||||
) x
|
||||
GROUP BY release_id
|
||||
)
|
||||
UPDATE releases r SET
|
||||
file_count = r.file_count + a.n,
|
||||
size = r.size + a.bytes,
|
||||
extensions = CASE
|
||||
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
||||
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
||||
search_files = CASE
|
||||
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
||||
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
||||
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
||||
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
||||
FROM added a
|
||||
WHERE r.id = a.release_id;
|
||||
|
||||
INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
||||
RETURN NULL;
|
||||
END $$`)
|
||||
|
||||
// The row triggers keep UPDATE and DELETE (renames, moves between releases); INSERT is the statement trigger.
|
||||
for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
|
||||
await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
|
||||
}
|
||||
await q(`CREATE TRIGGER files_release_extension AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
|
||||
await q(`CREATE TRIGGER files_release_search AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
|
||||
await q(`CREATE TRIGGER files_release_totals AFTER DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
await q(`CREATE TRIGGER files_zz_release_complete AFTER DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
await q(`CREATE TRIGGER files_release_insert AFTER INSERT ON files REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION files_release_insert_statement()`)
|
||||
}
|
||||
|
||||
async down() {
|
||||
const q = (sql: string) => this.db.rawQuery(sql)
|
||||
await q(`DROP TRIGGER IF EXISTS files_release_insert ON files`)
|
||||
for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
|
||||
await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
|
||||
}
|
||||
await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
|
||||
await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
|
||||
await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
|
||||
await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
|
||||
await q(`DROP FUNCTION IF EXISTS files_release_insert_statement()`)
|
||||
await q(`
|
||||
CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
|
||||
UPDATE releases r SET complete = c.complete
|
||||
FROM (SELECT release_complete(rid) AS complete) c
|
||||
WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
|
||||
$$`)
|
||||
await q(`DROP FUNCTION IF EXISTS recompute_release_complete(bigint)`)
|
||||
await q(`DROP TABLE IF EXISTS release_dirty`)
|
||||
}
|
||||
}
|
||||
@@ -391,6 +391,13 @@ export class PartSchema extends BaseModel {
|
||||
declare pcrc32: string | null
|
||||
}
|
||||
|
||||
export class ReleaseDirtySchema extends BaseModel {
|
||||
static $columns = ['releaseId'] as const
|
||||
$columns = ReleaseDirtySchema.$columns
|
||||
@column()
|
||||
declare releaseId: bigint | number
|
||||
}
|
||||
|
||||
export class ReleaseFileSchema extends BaseModel {
|
||||
static $columns = ['md5', 'md516K', 'name', 'par2FileId', 'releaseId', 'setId', 'size'] as const
|
||||
$columns = ReleaseFileSchema.$columns
|
||||
|
||||
Reference in new issue
Block a user