usenet-indexer/database/migrations/1768620765250_release_completeness_narrower.ts

import { BaseSchema } from '@adonisjs/lucid/schema'

/**
 * Release completeness (D24) for fewer releases, and without binaries
 * (LOG.md 2026-10-07 (7)).
 *
 * - Only releases whose completeness can change are noted for the sweep: a
 *   par2 listing, a file_total, or complete now (it can become incomplete).
 *   For any other release release_complete() is false by definition; noting
 *   every release a file went into made a sweep check 430,000 releases in
 *   51 minutes.
 * - release_complete() matches listed files by name against the files
 *   themselves (coalesce(real_name, filename)); the size match still goes
 *   through a binary when there is one. Seeding no longer creates binaries
 *   for most files and the GC deletes them, so a name only reachable through
 *   a binary was lost.
 * - Notes that don't qualify are dropped.
 */
export default class extends BaseSchema {
  async up() {
    const q = (sql: string) => this.db.rawQuery(sql)
    await q(`
      CREATE OR REPLACE FUNCTION release_completeness_matters(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
        SELECT EXISTS (SELECT 1 FROM releases r WHERE r.id = rid AND (r.file_total IS NOT NULL OR r.complete))
            OR EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid)
      $$`)
    await q(`
      CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$
        INSERT INTO release_dirty (release_id) SELECT rid WHERE rid IS NOT NULL AND release_completeness_matters(rid)
      $$`)
    await q(`
      CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean LANGUAGE sql STABLE AS $$
        WITH r AS (SELECT file_total, file_count FROM releases WHERE id = rid),
        -- The release's files, once, by index (read only when there's a par2 listing to check).
        have AS MATERIALIZED (
          SELECT b.size, coalesce(f.real_name, f.filename) AS name
          FROM files f LEFT JOIN binaries b ON b.file_id = f.id
          WHERE f.release_id = rid)
        SELECT coalesce((
          SELECT CASE
            WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total
            WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = rid) THEN NOT EXISTS (
              SELECT 1 FROM release_files rf
              WHERE rf.release_id = rid AND rf.size >= 65536
                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.size = rf.size)
                AND NOT EXISTS (SELECT 1 FROM have h WHERE h.name = rf.name))
            ELSE false END
          FROM r), false)
      $$`)
    await q(`
      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
        BEGIN
          WITH added AS (
            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
            FROM (
              SELECT release_id, size, name_extension(nm) AS e,
                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
            ) x
            GROUP BY release_id
          )
          UPDATE releases r SET
            file_count = r.file_count + a.n,
            size = r.size + a.bytes,
            extensions = CASE
              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
            search_files = CASE
              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
              ELSE nullif(left(concat_ws(' ', r.search_files, (
                SELECT string_agg(w, ' ') FROM unnest(a.words) w
                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
          FROM added a
          WHERE r.id = a.release_id;

          -- Only releases whose completeness can change (LOG.md 2026-10-07 (7)).
          INSERT INTO release_dirty (release_id)
            SELECT n.release_id FROM (SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL) n
            WHERE release_completeness_matters(n.release_id);
          INSERT INTO file_keys (key_hash)
            SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
            ON CONFLICT DO NOTHING;
          RETURN NULL;
        END $$`)
    await q(`DELETE FROM release_dirty d WHERE NOT release_completeness_matters(d.release_id)`)
  }

  async down() {
    // The previous definitions are in migrations …180, …190 and …220.
    throw new Error('Not reversible: re-run the earlier definitions by hand if needed.')
  }
}