import { BaseSchema } from '@adonisjs/lucid/schema' import env from '#start/env' /** * Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in * place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks, * inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives: * groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog * with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then: * - posters get an id; group_sets holds each distinct group combination once (~140k); releases is created anew * without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte * columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish * (null: a tail release still provisional). The category and judge triggers follow the new columns; * releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts * (migration 6200), which holds the name and every searchable string learned later. * - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once); * release_final() goes: a release is final once finished. * - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100). * - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per * release, not by search. */ export default class extends BaseSchema { async up() { this.defer(async (db) => { const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`) if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') { throw new Error( `The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead` ) } }) this.schema.raw(` TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets, release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty, judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`) // Spots matched to releases that are gone: the spot inspector matches them again. this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`) this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`) this.schema.raw(`DROP TABLE IF EXISTS fragments`) this.schema.raw(`DROP TABLE IF EXISTS range_blocks`) this.schema.raw( `DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)` ) this.schema.raw( `DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)` ) // Posters by id (a release's poster_id); searched by trigram as before (poster_search). this.schema.raw(`DROP TABLE IF EXISTS posters`) this.schema.raw(` CREATE TABLE posters ( id serial PRIMARY KEY, poster text NOT NULL UNIQUE )`) this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`) // Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names. this.schema.raw(` CREATE TABLE group_sets ( id serial PRIMARY KEY, groups jsonb NOT NULL UNIQUE )`) this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`) this.schema.raw(` CREATE TABLE releases ( id bigserial PRIMARY KEY, -- The upload's posting window (first and last article's Date). posted_at timestamptz, last_posted_at timestamptz, first_article bigint, last_article bigint, segments bigint NOT NULL DEFAULT 0, bytes bigint NOT NULL DEFAULT 0, -- The finish: the window closed on it (or the block ended); null while a tail release is provisional. settled_at timestamptz, inspected_at timestamptz, first_seen_at timestamptz NOT NULL DEFAULT now(), -- The owner group (null: a spot's release), the set of groups it was posted to, its poster. group_id integer, group_set_id integer NOT NULL, poster_id integer, file_total integer, files integer NOT NULL DEFAULT 0, chunks integer NOT NULL DEFAULT 0, category integer, -- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album. source smallint NOT NULL, readable boolean NOT NULL DEFAULT false, has_par2 boolean NOT NULL DEFAULT false, complete boolean, passworded boolean, sidecar_only boolean, -- Still open when its block ended: the block merger joins it with the neighbour block's part by identity. edge boolean NOT NULL DEFAULT false, -- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload). release_key bytea NOT NULL, identity_key bytea NOT NULL, name text NOT NULL, inner_name varchar(255) )`) this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`) this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`) // No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more. this.schema.raw( `CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)` ) this.schema.raw( `CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)` ) this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`) this.schema.raw( `CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge` ) this.schema.raw( `CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL` ) // The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets. this.schema.raw(` CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id), CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END); RETURN NEW; END $$`) this.schema.raw(` CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases FOR EACH ROW EXECUTE FUNCTION releases_set_category()`) // The judge's dirty marks (migration 4900), on the columns its rules read. this.schema.raw(` CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`) this.schema.raw(` CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`) this.schema.raw(` CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at OR old.complete IS DISTINCT FROM new.complete) EXECUTE FUNCTION judge_dirty_releases()`) // Chunks, rule counts and samples by the block that wrote them. this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`) this.schema.raw( `ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx` ) this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`) this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`) this.schema.raw( `ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index` ) // Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once. const hdd = env.get('CHUNK_TABLESPACE') if (hdd) { for (const table of ['release_inspections', 'probe_candidates']) { this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`) this.schema.raw(` DO $$ DECLARE i record; BEGIN FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}'); END LOOP; END $$`) } } } async down() { throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)') } }