usenet-indexer/database/migrations/1770000006000_voyager3_catalog.ts

import { BaseSchema } from '@adonisjs/lucid/schema'
import env from '#start/env'

/**
 * Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in
 * place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks,
 * inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives:
 * groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog
 * with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then:
 * - posters get an id; group_sets holds each distinct group combination once (~2,600); releases is created anew
 *   without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte
 *   columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish
 *   (null: a tail release still provisional). The category and judge triggers follow the new columns;
 *   releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts
 *   (migration 6200), which holds the name and every searchable string learned later.
 * - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once);
 *   release_final() goes: a release is final once finished.
 * - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100).
 * - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per
 *   release, not by search.
 */
export default class extends BaseSchema {
  async up() {
    this.defer(async (db) => {
      const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`)
      if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') {
        throw new Error(
          `The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead`
        )
      }
    })
    this.schema.raw(`
      TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets,
        release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty,
        judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`)
    // Spots matched to releases that are gone: the spot inspector matches them again.
    this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`)
    this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`)

    this.schema.raw(`DROP TABLE IF EXISTS fragments`)
    this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
    this.schema.raw(
      `DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)`
    )
    this.schema.raw(
      `DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`
    )

    // Posters by id (a release's poster_id); searched by trigram as before (poster_search).
    this.schema.raw(`DROP TABLE IF EXISTS posters`)
    this.schema.raw(`
      CREATE TABLE posters (
        id serial PRIMARY KEY,
        poster text NOT NULL UNIQUE
      )`)
    this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
    // Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names.
    this.schema.raw(`
      CREATE TABLE group_sets (
        id serial PRIMARY KEY,
        groups jsonb NOT NULL UNIQUE
      )`)

    this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`)
    this.schema.raw(`
      CREATE TABLE releases (
        id bigserial PRIMARY KEY,
        -- The upload's posting window (first and last article's Date).
        posted_at timestamptz,
        last_posted_at timestamptz,
        first_article bigint,
        last_article bigint,
        segments bigint NOT NULL DEFAULT 0,
        bytes bigint NOT NULL DEFAULT 0,
        -- The finish: the window closed on it (or the block ended); null while a tail release is provisional.
        settled_at timestamptz,
        inspected_at timestamptz,
        first_seen_at timestamptz NOT NULL DEFAULT now(),
        -- The owner group (null: a spot's release), the set of groups it was posted to, its poster.
        group_id integer,
        group_set_id integer NOT NULL,
        poster_id integer,
        file_total integer,
        files integer NOT NULL DEFAULT 0,
        chunks integer NOT NULL DEFAULT 0,
        category integer,
        -- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album.
        source smallint NOT NULL,
        readable boolean NOT NULL DEFAULT false,
        has_par2 boolean NOT NULL DEFAULT false,
        complete boolean,
        passworded boolean,
        sidecar_only boolean,
        -- Still open when its block ended: the block merger joins it with the neighbour block's part by identity.
        edge boolean NOT NULL DEFAULT false,
        -- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload).
        release_key bytea NOT NULL,
        identity_key bytea NOT NULL,
        name text NOT NULL,
        inner_name varchar(255)
      )`)
    this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`)
    this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`)
    // No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more.
    this.schema.raw(
      `CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`
    )
    this.schema.raw(
      `CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)`
    )
    this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`)
    this.schema.raw(
      `CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge`
    )
    this.schema.raw(
      `CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL`
    )

    // The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets.
    this.schema.raw(`
      CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
      BEGIN
        NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id),
                                             CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
        RETURN NEW;
      END $$`)
    this.schema.raw(`
      CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases
      FOR EACH ROW EXECUTE FUNCTION releases_set_category()`)
    // The judge's dirty marks (migration 4900), on the columns its rules read.
    this.schema.raw(`
      CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows
      FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
    this.schema.raw(`
      CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows
      FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
    this.schema.raw(`
      CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
      WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source
        OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded
        OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable
        OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id
        OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article
        OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at
        OR old.complete IS DISTINCT FROM new.complete)
      EXECUTE FUNCTION judge_dirty_releases()`)

    // Chunks, rule counts and samples by the block that wrote them.
    this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`)
    this.schema.raw(
      `ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx`
    )
    this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`)
    this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`)
    this.schema.raw(
      `ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index`
    )

    // Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once.
    const hdd = env.get('CHUNK_TABLESPACE')
    if (hdd) {
      for (const table of ['release_inspections', 'probe_candidates']) {
        this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`)
        this.schema.raw(`
          DO $$ DECLARE i record; BEGIN
            FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP
              EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}');
            END LOOP;
          END $$`)
      }
    }
  }

  async down() {
    throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)')
  }
}