Voyager 3 schema: the slim catalog (migration 6000, the reset) and blocks (6100)

releases rebuilt without its text copies: poster_id, group_set_id, bytea keys, source smallint, edge, settled_at as the finish. posters by id, group_sets. blocks as the lease unit, ranges.block_id. fragments, range_blocks, release_final, the settler and the fragment merger gone; release_chunks, rule_counts and rule_samples keyed by block. Cold tables on CHUNK_TABLESPACE. Migration 6000 truncates the catalog and refuses without VOYAGER3_RESET=1. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-25 13:44:54 +01:00
1 parent 1a448f37f9
commit 346493b6c3
@@ -0,0 +1,181 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import env from '#start/env'
+
+/**
+ * Voyager 3's catalog (REBUILD.md, "Voyager 3: window indexing", 4 and 7), built from a reset, not converted in
+ * place. This migration IS the reset: it empties every table the indexing recomputes (releases, chunks,
+ * inspections, candidates, par2 data, file hashes, verdicts, merges, rule counts and samples; what survives:
+ * groups, ranges, the corpus, probe_cache, spots, nntp_shares, nodes, users), so it refuses to run on a catalog
+ * with releases unless VOYAGER3_RESET=1 (the user's go-ahead; resets are paused since 2026-10-12). Then:
+ * - posters get an id; group_sets holds each distinct group combination once (~2,600); releases is created anew
+ *   without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte
+ *   columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish
+ *   (null: a tail release still provisional). The category and judge triggers follow the new columns;
+ *   releases_category_newest_idx (5.3 GB, used once) goes.
+ * - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once);
+ *   release_final() goes: a release is final once finished.
+ * - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100).
+ * - release_inspections and probe_candidates move to CHUNK_TABLESPACE (the HDD) when set: written once, read per
+ *   release, not by search.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      const { rows } = await db.rawQuery(`SELECT count(*)::int AS n FROM releases`)
+      if (rows[0].n > 0 && process.env.VOYAGER3_RESET !== '1') {
+        throw new Error(
+          `The voyager 3 catalog migration empties the catalog (${rows[0].n} releases): run it with VOYAGER3_RESET=1 after the go-ahead`
+        )
+      }
+    })
+    this.schema.raw(`
+      TRUNCATE releases, release_chunks, release_inspections, probe_candidates, file_hashes, par2_files, par2_sets,
+        release_verdicts, release_merges, release_spots, release_passwords, rule_counts, rule_samples, judge_dirty,
+        judge_full, fold_candidates, posters, worker_cursors RESTART IDENTITY`)
+    // Spots matched to releases that are gone: the spot inspector matches them again.
+    this.schema.raw(`UPDATE spots SET release_id = NULL WHERE release_id IS NOT NULL`)
+    this.schema.raw(`UPDATE spot_files SET release_id = NULL WHERE release_id IS NOT NULL`)
+
+    this.schema.raw(`DROP TABLE IF EXISTS fragments`)
+    this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
+    this.schema.raw(
+      `DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz, boolean)`
+    )
+    this.schema.raw(
+      `DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`
+    )
+
+    // Posters by id (a release's poster_id); searched by trigram as before (poster_search).
+    this.schema.raw(`DROP TABLE IF EXISTS posters`)
+    this.schema.raw(`
+      CREATE TABLE posters (
+        id serial PRIMARY KEY,
+        poster text NOT NULL UNIQUE
+      )`)
+    this.schema.raw(`CREATE INDEX posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
+    // Each distinct combination of groups a release was posted to, once: a sorted jsonb array of names.
+    this.schema.raw(`
+      CREATE TABLE group_sets (
+        id serial PRIMARY KEY,
+        groups jsonb NOT NULL UNIQUE
+      )`)
+
+    this.schema.raw(`DROP TABLE IF EXISTS releases CASCADE`)
+    this.schema.raw(`
+      CREATE TABLE releases (
+        id bigserial PRIMARY KEY,
+        -- The upload's posting window (first and last article's Date).
+        posted_at timestamptz,
+        last_posted_at timestamptz,
+        first_article bigint,
+        last_article bigint,
+        segments bigint NOT NULL DEFAULT 0,
+        bytes bigint NOT NULL DEFAULT 0,
+        -- The finish: the window closed on it (or the block ended); null while a tail release is provisional.
+        settled_at timestamptz,
+        inspected_at timestamptz,
+        first_seen_at timestamptz NOT NULL DEFAULT now(),
+        -- The owner group (null: a spot's release), the set of groups it was posted to, its poster.
+        group_id integer,
+        group_set_id integer NOT NULL,
+        poster_id integer,
+        file_total integer,
+        files integer NOT NULL DEFAULT 0,
+        chunks integer NOT NULL DEFAULT 0,
+        category integer,
+        -- lib/release/source.ts: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6 album.
+        source smallint NOT NULL,
+        readable boolean NOT NULL DEFAULT false,
+        has_par2 boolean NOT NULL DEFAULT false,
+        complete boolean,
+        passworded boolean,
+        sidecar_only boolean,
+        -- Still open when its block ended: the block merger joins it with the neighbour block's part by identity.
+        edge boolean NOT NULL DEFAULT false,
+        -- SHA-1s (lib/release/identity.ts): the identity (poster, name, group), and the release (identity and upload).
+        release_key bytea NOT NULL,
+        identity_key bytea NOT NULL,
+        name text NOT NULL,
+        inner_name varchar(255)
+      )`)
+    this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`)
+    this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`)
+    this.schema.raw(
+      `CREATE INDEX releases_name_trgm ON releases USING gin (name gin_trgm_ops) WITH (fastupdate = on, gin_pending_list_limit = 65536)`
+    )
+    this.schema.raw(
+      `CREATE INDEX releases_inner_name_trgm ON releases USING gin (inner_name gin_trgm_ops) WHERE inner_name IS NOT NULL`
+    )
+    this.schema.raw(
+      `CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)`
+    )
+    this.schema.raw(
+      `CREATE INDEX releases_poster_newest_idx ON releases (poster_id, posted_at DESC NULLS LAST, id DESC)`
+    )
+    this.schema.raw(`CREATE INDEX releases_passworded_idx ON releases (id) WHERE passworded`)
+    this.schema.raw(
+      `CREATE INDEX releases_edge_idx ON releases (group_id, identity_key) WHERE edge`
+    )
+    this.schema.raw(
+      `CREATE INDEX releases_provisional_idx ON releases (id) WHERE settled_at IS NULL`
+    )
+
+    // The category from the name, the inner name, or the groups (migration 5200), the groups now in group_sets.
+    this.schema.raw(`
+      CREATE OR REPLACE FUNCTION releases_set_category() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        NEW.category := release_category_for(NEW.name, NEW.inner_name, (SELECT groups FROM group_sets WHERE id = NEW.group_set_id),
+                                             CASE WHEN TG_OP = 'UPDATE' THEN OLD.category ELSE NEW.category END);
+        RETURN NEW;
+      END $$`)
+    this.schema.raw(`
+      CREATE TRIGGER releases_category BEFORE INSERT OR UPDATE OF name, inner_name, group_set_id ON releases
+      FOR EACH ROW EXECUTE FUNCTION releases_set_category()`)
+    // The judge's dirty marks (migration 4900), on the columns its rules read.
+    this.schema.raw(`
+      CREATE TRIGGER judge_dirty_insert AFTER INSERT ON releases REFERENCING NEW TABLE AS new_rows
+      FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
+    this.schema.raw(`
+      CREATE TRIGGER judge_dirty_delete AFTER DELETE ON releases REFERENCING OLD TABLE AS old_rows
+      FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_releases()`)
+    this.schema.raw(`
+      CREATE TRIGGER judge_dirty_update AFTER UPDATE ON releases FOR EACH ROW
+      WHEN (old.name IS DISTINCT FROM new.name OR old.poster_id IS DISTINCT FROM new.poster_id OR old.source IS DISTINCT FROM new.source
+        OR old.file_total IS DISTINCT FROM new.file_total OR old.passworded IS DISTINCT FROM new.passworded
+        OR old.settled_at IS DISTINCT FROM new.settled_at OR old.readable IS DISTINCT FROM new.readable
+        OR old.inner_name IS DISTINCT FROM new.inner_name OR old.group_id IS DISTINCT FROM new.group_id
+        OR old.first_article IS DISTINCT FROM new.first_article OR old.last_article IS DISTINCT FROM new.last_article
+        OR old.posted_at IS DISTINCT FROM new.posted_at OR old.last_posted_at IS DISTINCT FROM new.last_posted_at
+        OR old.complete IS DISTINCT FROM new.complete)
+      EXECUTE FUNCTION judge_dirty_releases()`)
+
+    // Chunks, rule counts and samples by the block that wrote them.
+    this.schema.raw(`ALTER TABLE release_chunks RENAME COLUMN range_id TO block_id`)
+    this.schema.raw(
+      `ALTER INDEX IF EXISTS release_chunks_range_idx RENAME TO release_chunks_block_idx`
+    )
+    this.schema.raw(`ALTER TABLE rule_counts RENAME COLUMN range_id TO block_id`)
+    this.schema.raw(`ALTER TABLE rule_samples RENAME COLUMN range_id TO block_id`)
+    this.schema.raw(
+      `ALTER INDEX IF EXISTS rule_samples_range_id_index RENAME TO rule_samples_block_id_index`
+    )
+
+    // Cold tables to the HDD (CHUNK_TABLESPACE), with their indexes; empty now, so at once.
+    const hdd = env.get('CHUNK_TABLESPACE')
+    if (hdd) {
+      for (const table of ['release_inspections', 'probe_candidates']) {
+        this.schema.raw(`ALTER TABLE ${table} SET TABLESPACE ${hdd}`)
+        this.schema.raw(`
+          DO $$ DECLARE i record; BEGIN
+            FOR i IN SELECT indexname FROM pg_indexes WHERE schemaname = 'public' AND tablename = '${table}' LOOP
+              EXECUTE format('ALTER INDEX %I SET TABLESPACE %I', i.indexname, '${hdd}');
+            END LOOP;
+          END $$`)
+      }
+    }
+  }
+
+  async down() {
+    throw new Error('The voyager 3 catalog migration cannot be undone (it emptied the catalog)')
+  }
+}
@@ -0,0 +1,61 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Blocks (REBUILD.md, "Voyager 3: window indexing", 1): what a worker leases. A block is a run of consecutive
+ * ranges of one group covering at least 30 days of posting time (planned from ranges.est_date and posted_low/high),
+ * read in article order through the 24 h window; blocks of a group run in parallel. Ranges stay the unit the
+ * prefetcher fetches (the spool, the corpus) and point at their block; the lease, resume point, heartbeat and
+ * counts move here. Part of the reset (migration 6000): every range is read again, from the corpus where stored.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.raw(`
+      CREATE TABLE blocks (
+        id bigserial PRIMARY KEY,
+        group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
+        -- The first range's start and the last range's end.
+        start_id bigint NOT NULL,
+        end_id bigint NOT NULL,
+        -- The resume point: the first article not stored yet (batches are refused out of order, as ranges' were).
+        next_id bigint NOT NULL,
+        -- The posting time the block covers: estimated when planned, the worker's measured span once done.
+        posted_low timestamptz,
+        posted_high timestamptz,
+        -- planned, leased, done, failed.
+        status varchar(8) NOT NULL DEFAULT 'planned',
+        -- As ranges': the tail before everything, then the history by era.
+        priority integer NOT NULL DEFAULT 0,
+        node_id integer,
+        attempts integer NOT NULL DEFAULT 0,
+        rule_version integer,
+        headers bigint NOT NULL DEFAULT 0,
+        releases integer NOT NULL DEFAULT 0,
+        leased_at timestamptz,
+        heartbeat_at timestamptz,
+        done_at timestamptz,
+        error text,
+        created_at timestamptz NOT NULL DEFAULT now(),
+        UNIQUE (group_id, start_id)
+      )`)
+    this.schema.raw(
+      `CREATE INDEX blocks_todo_idx ON blocks (priority, id) WHERE status IN ('planned', 'leased')`
+    )
+    this.schema.raw(`CREATE INDEX blocks_leased_idx ON blocks (leased_at) WHERE status = 'leased'`)
+    this.schema.raw(
+      `ALTER TABLE ranges ADD COLUMN IF NOT EXISTS block_id bigint REFERENCES blocks (id) ON DELETE SET NULL`
+    )
+    this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_block_idx ON ranges (block_id)`)
+    // The reset: every range planned again from its start (the corpus serves the stored ones). Priorities, capture,
+    // est_date, stored_at, plain_chunks and the measured posted_low/high stay.
+    this.schema.raw(`
+      UPDATE ranges SET status = 'planned', next_id = start_id, node_id = NULL, attempts = 0, spooled_chunks = 0,
+        spooled_at = NULL, leased_at = NULL, heartbeat_at = NULL, done_at = NULL, headers = 0, error = NULL,
+        rule_version = NULL
+      WHERE status <> 'planned' OR next_id <> start_id OR spooled_chunks <> 0 OR node_id IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS block_id`)
+    this.schema.raw(`DROP TABLE IF EXISTS blocks`)
+  }
+}