usenet-indexer/database/migrations/1768620765130_release_extensions.ts

import { BaseSchema } from '@adonisjs/lucid/schema'

const VISIBLE = `(file_count > 0 OR source = 'spot')`
const BATCH = 20_000

/**
 * `releases.extensions`: the file types in a release (`{mkv,nfo,par2}`), from
 * its files' names and its archive listing, for the search's extension
 * filter (LOG.md 2026-10-04 (3)). search_text can't serve it: file names
 * enter it as stems without extension, and matching the long, TOASTed text
 * took seconds per search.
 */
export default class extends BaseSchema {
  static disableTransactions = true

  async up() {
    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)

    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS extensions text[]`)
    // Two to four characters with a letter (mkv, flac, 7z, par2); split
    // archive volumes count as their archive type (.7z.001, .r00).
    await q(`
      CREATE OR REPLACE FUNCTION name_extension(n text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
        SELECT CASE
          WHEN lower(n) ~ '\\.(7z|zip)\\.[0-9]{3}$' THEN substring(lower(n) from '\\.(7z|zip)\\.[0-9]{3}$')
          WHEN lower(n) ~ '\\.r[0-9]{2,3}$' THEN 'rar'
          ELSE substring(lower(n) from '\\.([a-z0-9]*[a-z][a-z0-9]*)$')
        END
      $$`)
    await q(`
      CREATE OR REPLACE FUNCTION listing_extensions(contents jsonb) RETURNS text[] IMMUTABLE LANGUAGE sql AS $$
        SELECT array(SELECT DISTINCT e FROM (
          SELECT name_extension(c->>'name') AS e
          FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) c) x
        WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 ORDER BY e)
      $$`)

    // A new archive listing adds its entries' types.
    await q(`
      CREATE OR REPLACE FUNCTION releases_extensions() RETURNS trigger LANGUAGE plpgsql AS $$
      BEGIN
        IF TG_OP = 'INSERT' OR NEW.contents IS DISTINCT FROM OLD.contents THEN
          NEW.extensions := nullif(array(
            SELECT DISTINCT e FROM unnest(coalesce(NEW.extensions, '{}') || listing_extensions(NEW.contents)) e ORDER BY e), '{}');
        END IF;
        RETURN NEW;
      END $$`)
    await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
    await q(`CREATE TRIGGER releases_extensions BEFORE INSERT OR UPDATE OF contents ON releases FOR EACH ROW EXECUTE FUNCTION releases_extensions()`)

    // A file adds its type to its release (once: most files of a release share it).
    await q(`
      CREATE OR REPLACE FUNCTION files_release_extension() RETURNS trigger LANGUAGE plpgsql AS $$
      DECLARE
        e text := name_extension(coalesce(NEW.real_name, NEW.filename));
      BEGIN
        IF NEW.release_id IS NULL OR e IS NULL OR length(e) NOT BETWEEN 2 AND 4 THEN RETURN NULL; END IF;
        UPDATE releases SET extensions = array(SELECT DISTINCT x FROM unnest(coalesce(extensions, '{}') || e) x ORDER BY x)
        WHERE id = NEW.release_id AND NOT (coalesce(extensions, '{}') @> ARRAY[e]);
        RETURN NULL;
      END $$`)
    await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)

    // Fill: file types per release in one sequential pass over files (random
    // reads per release would take over an hour on the HDD), then the
    // visible releases in batches by id, adding their listing's types.
    await q(`DROP TABLE IF EXISTS file_extensions_tmp`)
    await q(`
      CREATE UNLOGGED TABLE file_extensions_tmp AS
      SELECT release_id, array_agg(DISTINCT e) AS exts FROM (
        SELECT release_id, name_extension(coalesce(real_name, filename)) AS e FROM files WHERE release_id IS NOT NULL) x
      WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 GROUP BY release_id`)
    await q(`CREATE UNIQUE INDEX ON file_extensions_tmp (release_id)`)
    let last = 0
    let done = 0
    while (true) {
      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
      if (rows.length === 0) break
      const to = Number(rows[rows.length - 1].id)
      await q(
        `UPDATE releases r SET extensions = x.exts FROM (
           SELECT r2.id, array(SELECT DISTINCT e FROM unnest(coalesce(t.exts, '{}') || listing_extensions(r2.contents)) e ORDER BY e) AS exts
           FROM releases r2 LEFT JOIN file_extensions_tmp t ON t.release_id = r2.id
           WHERE r2.id > ? AND r2.id <= ? AND (r2.file_count > 0 OR r2.source = 'spot')) x
         WHERE r.id = x.id AND cardinality(x.exts) > 0 AND r.extensions IS DISTINCT FROM x.exts`,
        [last, to]
      )
      last = to
      done += rows.length
      if (done % 500_000 < BATCH) console.log(`Extensions for ${done} releases.`)
    }
    await q(`DROP TABLE file_extensions_tmp`)

    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_extensions_gin ON releases USING gin (extensions) WHERE ${VISIBLE}`)
  }

  async down() {
    const q = (sql: string) => this.db.rawQuery(sql)
    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_extensions_gin`)
    await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`)
    await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`)
    await q(`DROP FUNCTION IF EXISTS files_release_extension()`)
    await q(`DROP FUNCTION IF EXISTS releases_extensions()`)
    await q(`DROP FUNCTION IF EXISTS listing_extensions(jsonb)`)
    await q(`DROP FUNCTION IF EXISTS name_extension(text)`)
    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS extensions`)
  }
}