178 lines
9.5 KiB
TypeScript
178 lines
9.5 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
import env from '#start/env'
|
|
|
|
/**
|
|
* Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in
|
|
* place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks,
|
|
* inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives:
|
|
* groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog
|
|
* with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then:
|
|
* - posters get an id; group_sets holds each distinct group combination once (~140k); releases is created anew
|
|
* without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte
|
|
* columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish
|
|
* (null: a tail release still provisional). The category and judge triggers follow the new columns;
|
|
* releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts
|
|
* (migration 6200), which holds the name and every searchable string learned later.
|
|
* - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once);
|
|
* release_final() goes: a release is final once finished.
|
|
* - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100).
|
|
* - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per
|
|
* release, not by search.
|
|
*/
|
|
export default class extends BaseSchema {
|
|
async up() {
|
|
this.defer(async (db) => {
|
|
const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`)
|
|
if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') {
|
|
throw new Error(
|
|
`The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead`
|
|
)
|
|
}
|
|
})
|
|
this.schema.raw(`
|
|
TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets,
|
|
release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty,
|
|
judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`)
|
|
// Spots matched to releases that are gone: the spot inspector matches them again.
|
|
this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`)
|
|
this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`)
|
|
|
|
this.schema.raw(`DROP TABLE IF EXISTS fragments`)
|
|
this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
|
|
this.schema.raw(
|
|
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)`
|
|
)
|
|
this.schema.raw(
|
|
`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`
|
|
)
|
|
|
|
// Posters by id (a release's poster_id); searched by trigram as before (poster_search).
|
|
this.schema.raw(`DROP TABLE IF EXISTS posters`)
|
|
this.schema.raw(`
|
|
CREATE TABLE posters (
|
|
id serial PRIMARY KEY,
|
|
poster text NOT NULL UNIQUE
|
|
)`)
|
|
this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
|
|
// Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names.
|
|
this.schema.raw(`
|
|
CREATE TABLE group_sets (
|
|
id serial PRIMARY KEY,
|
|
groups jsonb NOT NULL UNIQUE
|
|
)`)
|
|
|
|
this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`)
|
|
this.schema.raw(`
|
|
CREATE TABLE releases (
|
|
id bigserial PRIMARY KEY,
|
|
-- The upload's posting window (first and last article's Date).
|
|
posted_at timestamptz,
|
|
last_posted_at timestamptz,
|
|
first_article bigint,
|
|
last_article bigint,
|
|
segments bigint NOT NULL DEFAULT 0,
|
|
bytes bigint NOT NULL DEFAULT 0,
|
|
-- The finish: the window closed on it (or the block ended); null while a tail release is provisional.
|
|
settled_at timestamptz,
|
|
inspected_at timestamptz,
|
|
first_seen_at timestamptz NOT NULL DEFAULT now(),
|
|
-- The owner group (null: a spot's release), the set of groups it was posted to, its poster.
|
|
group_id integer,
|
|
group_set_id integer NOT NULL,
|
|
poster_id integer,
|
|
file_total integer,
|
|
files integer NOT NULL DEFAULT 0,
|
|
chunks integer NOT NULL DEFAULT 0,
|
|
category integer,
|
|
-- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album.
|
|
source smallint NOT NULL,
|
|
readable boolean NOT NULL DEFAULT false,
|
|
has_par2 boolean NOT NULL DEFAULT false,
|
|
complete boolean,
|
|
passworded boolean,
|
|
sidecar_only boolean,
|
|
-- Still open when its block ended: the block merger joins it with the neighbour block's part by identity.
|
|
edge boolean NOT NULL DEFAULT false,
|
|
-- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload).
|
|
release_key bytea NOT NULL,
|
|
identity_key bytea NOT NULL,
|
|
name text NOT NULL,
|
|
inner_name varchar(255)
|
|
)`)
|
|
this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`)
|
|
this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`)
|
|
// No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more.
|
|
this.schema.raw(
|
|
`CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`
|
|
)
|
|
this.schema.raw(
|
|
`CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)`
|
|
)
|
|
this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`)
|
|
this.schema.raw(
|
|
`CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge`
|
|
)
|
|
this.schema.raw(
|
|
`CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL`
|
|
)
|
|
|
|
// The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets.
|
|
this.schema.raw(`
|
|
CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id),
|
|
CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
|
|
RETURN NEW;
|
|
END $$`)
|
|
this.schema.raw(`
|
|
CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases
|
|
FOR EACH ROW EXECUTE FUNCTION releases_set_category()`)
|
|
// The judge's dirty marks (migration 4900), on the columns its rules read.
|
|
this.schema.raw(`
|
|
CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows
|
|
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
|
|
this.schema.raw(`
|
|
CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows
|
|
FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
|
|
this.schema.raw(`
|
|
CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
|
|
WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source
|
|
OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded
|
|
OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable
|
|
OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id
|
|
OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article
|
|
OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at
|
|
OR old.complete IS DISTINCT FROM new.complete)
|
|
EXECUTE FUNCTION judge_dirty_releases()`)
|
|
|
|
// Chunks, rule counts and samples by the block that wrote them.
|
|
this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`)
|
|
this.schema.raw(
|
|
`ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx`
|
|
)
|
|
this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`)
|
|
this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`)
|
|
this.schema.raw(
|
|
`ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index`
|
|
)
|
|
|
|
// Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once.
|
|
const hdd = env.get('CHUNK_TABLESPACE')
|
|
if (hdd) {
|
|
for (const table of ['release_inspections', 'probe_candidates']) {
|
|
this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`)
|
|
this.schema.raw(`
|
|
DO $$ DECLARE i record; BEGIN
|
|
FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP
|
|
EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}');
|
|
END LOOP;
|
|
END $$`)
|
|
}
|
|
}
|
|
}
|
|
|
|
async down() {
|
|
throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)')
|
|
}
|
|
}
|