usenet-indexer/database/migrations/1768620765190_release_totals_per_statement.ts

import { BaseSchema } from '@adonisjs/lucid/schema'

/**
 * Cheaper release bookkeeping for file inserts (LOG.md 2026-10-06 (6)).
 *
 * Every inserted file fired four row triggers, each updating its release:
 * file_count/size, search_files, extensions and completeness, so a seed
 * batch of thousands of files rewrote its releases thousands of times (a
 * quarter of Postgres' active time on homelab). Now one statement-level
 * trigger updates each release once per INSERT, with all four.
 *
 * Completeness (D24), the expensive part, is no longer computed in triggers:
 * refresh_release_complete() only notes the release in release_dirty, and
 * gc:scheduler recomputes the noted releases every 30 minutes
 * (recompute_release_complete). Counts, sizes, search text and extensions
 * stay current: code decides on them right away (weak releases, deleting
 * empty releases).
 *
 * Also an index on release_files (release_id, size) for the par2 size
 * match's "listed once in its set" check (IngestService.matchPar2Releases).
 */
export default class extends BaseSchema {
  async up() {
    const q = (sql: string) => this.db.rawQuery(sql)
    // Swapping triggers locks files and its partitions: give up rather than queue every writer behind it.
    await q(`SET LOCAL lock_timeout = '2min'`)
    // Built CONCURRENTLY by hand on homelab (2026-10-06); IF NOT EXISTS keeps that one.
    await q(`CREATE INDEX IF NOT EXISTS release_files_release_size_idx ON release_files (release_id, size)`)

    // Append-only (no key): writers never wait on each other or on the sweep.
    await q(`CREATE TABLE IF NOT EXISTS release_dirty (release_id bigint NOT NULL)`)
    await q(`
      CREATE OR REPLACE FUNCTION recompute_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
        UPDATE releases r SET complete = c.complete
        FROM (SELECT release_complete(rid) AS complete) c
        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
      $$`)
    // Every trigger that called this (files, release_files, binaries.size, releases.file_total) now just notes the release.
    await q(`
      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
        INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL
      $$`)

    await q(`
      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
        BEGIN
          WITH added AS (
            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
            FROM (
              SELECT release_id, size, name_extension(nm) AS e,
                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
            ) x
            GROUP BY release_id
          )
          UPDATE releases r SET
            file_count = r.file_count + a.n,
            size = r.size + a.bytes,
            extensions = CASE
              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
            search_files = CASE
              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
              ELSE nullif(left(concat_ws(' ', r.search_files, (
                SELECT string_agg(w, ' ') FROM unnest(a.words) w
                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
          FROM added a
          WHERE r.id = a.release_id;

          INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
          RETURN NULL;
        END $$`)

    // The row triggers keep UPDATE and DELETE (renames, moves between releases); INSERT is the statement trigger.
    for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
      await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
    }
    await q(`CREATE TRIGGER files_release_extension AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
    await q(`CREATE TRIGGER files_release_search AFTER UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
    await q(`CREATE TRIGGER files_release_totals AFTER DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
    await q(`CREATE TRIGGER files_zz_release_complete AFTER DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
    await q(`CREATE TRIGGER files_release_insert AFTER INSERT ON files REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION files_release_insert_statement()`)
  }

  async down() {
    const q = (sql: string) => this.db.rawQuery(sql)
    await q(`DROP TRIGGER IF EXISTS files_release_insert ON files`)
    for (const name of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
      await q(`DROP TRIGGER IF EXISTS ${name} ON files`)
    }
    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
    await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
    await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
    await q(`DROP FUNCTION IF EXISTS files_release_insert_statement()`)
    await q(`
      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
        UPDATE releases r SET complete = c.complete
        FROM (SELECT release_complete(rid) AS complete) c
        WHERE r.id = rid AND r.complete IS DISTINCT FROM c.complete
      $$`)
    await q(`DROP FUNCTION IF EXISTS recompute_release_complete(bigint)`)
    await q(`DROP TABLE IF EXISTS release_dirty`)
  }
}