import { BaseSchema } from '@adonisjs/lucid/schema' /** * Blocks (REBUILD.md, "Voyager 3: window indexing", 1): what a worker leases. A block is a run of consecutive * ranges of one group covering at least 30 days of posting time (planned from ranges.est_date and posted_low/high), * read in article order through the 24 h window; blocks of a group run in parallel. Ranges stay the unit the * prefetcher fetches (the spool, the corpus) and point at their block; the lease, resume point, heartbeat and * counts move here. Part of the reset (migration 6000): every range is read again, from the corpus where stored. */ export default class extends BaseSchema { async up() { this.schema.raw(` CREATE TABLE blocks ( id bigserial PRIMARY KEY, group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE, -- The first range's start and the last range's end. start_id bigint NOT NULL, end_id bigint NOT NULL, -- The resume point: the first article not stored yet (batches are refused out of order, as ranges' were). next_id bigint NOT NULL, -- The posting time the block covers: estimated when planned, the worker's measured span once done. posted_low timestamptz, posted_high timestamptz, -- planned, leased, done, failed. status varchar(8) NOT NULL DEFAULT 'planned', -- As ranges': the tail before everything, then the history by era. priority integer NOT NULL DEFAULT 0, node_id integer, attempts integer NOT NULL DEFAULT 0, rule_version integer, headers bigint NOT NULL DEFAULT 0, releases integer NOT NULL DEFAULT 0, leased_at timestamptz, heartbeat_at timestamptz, done_at timestamptz, error text, created_at timestamptz NOT NULL DEFAULT now(), UNIQUE (group_id, start_id) )`) this.schema.raw( `CREATE INDEX blocks_todo_idx ON blocks (priority, id) WHERE status IN ('planned', 'leased')` ) this.schema.raw(`CREATE INDEX blocks_leased_idx ON blocks (leased_at) WHERE status = 'leased'`) this.schema.raw( `ALTER TABLE ranges ADD COLUMN IF NOT EXISTS block_id bigint REFERENCES blocks (id) ON DELETE SET NULL` ) this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_block_idx ON ranges (block_id)`) // The reset: every range planned again from its start (the corpus serves the stored ones). Priorities, capture, // est_date, stored_at, plain_chunks and the measured posted_low/high stay. this.schema.raw(` UPDATE ranges SET status = 'planned', next_id = start_id, node_id = NULL, attempts = 0, spooled_chunks = 0, spooled_at = NULL, leased_at = NULL, heartbeat_at = NULL, done_at = NULL, headers = 0, error = NULL, rule_version = NULL WHERE status <> 'planned' OR next_id <> start_id OR spooled_chunks <> 0 OR node_id IS NOT NULL`) } async down() { this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS block_id`) this.schema.raw(`DROP TABLE IF EXISTS blocks`) } }