Daan MeijerandClaude Fable 5.1
346493b6c3
releases rebuilt without its text copies: poster_id, group_set_id, bytea keys, source smallint, edge, settled_at as the finish. posters by id, group_sets. blocks as the lease unit, ranges.block_id. fragments, range_blocks, release_final, the settler and the fragment merger gone; release_chunks, rule_counts and rule_samples keyed by block. Cold tables on CHUNK_TABLESPACE. Migration 6000 truncates the catalog and refuses without VOYAGER3_RESET=1. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
62 lines
3.0 KiB
TypeScript
62 lines
3.0 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
|
|
/**
|
|
* Blocks (REBUILD.md, "Voyager 3: window indexing", 1): what a worker leases. A block is a run of consecutive
|
|
* ranges of one group covering at least 30 days of posting time (planned from ranges.est_date and posted_low/high),
|
|
* read in article order through the 24 h window; blocks of a group run in parallel. Ranges stay the unit the
|
|
* prefetcher fetches (the spool, the corpus) and point at their block; the lease, resume point, heartbeat and
|
|
* counts move here. Part of the reset (migration 6000): every range is read again, from the corpus where stored.
|
|
*/
|
|
export default class extends BaseSchema {
|
|
async up() {
|
|
this.schema.raw(`
|
|
CREATE TABLE blocks (
|
|
id bigserial PRIMARY KEY,
|
|
group_id integer NOT NULL REFERENCES groups (id) ON DELETE CASCADE,
|
|
-- The first range's start and the last range's end.
|
|
start_id bigint NOT NULL,
|
|
end_id bigint NOT NULL,
|
|
-- The resume point: the first article not stored yet (batches are refused out of order, as ranges' were).
|
|
next_id bigint NOT NULL,
|
|
-- The posting time the block covers: estimated when planned, the worker's measured span once done.
|
|
posted_low timestamptz,
|
|
posted_high timestamptz,
|
|
-- planned, leased, done, failed.
|
|
status varchar(8) NOT NULL DEFAULT 'planned',
|
|
-- As ranges': the tail before everything, then the history by era.
|
|
priority integer NOT NULL DEFAULT 0,
|
|
node_id integer,
|
|
attempts integer NOT NULL DEFAULT 0,
|
|
rule_version integer,
|
|
headers bigint NOT NULL DEFAULT 0,
|
|
releases integer NOT NULL DEFAULT 0,
|
|
leased_at timestamptz,
|
|
heartbeat_at timestamptz,
|
|
done_at timestamptz,
|
|
error text,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE (group_id, start_id)
|
|
)`)
|
|
this.schema.raw(
|
|
`CREATE INDEX blocks_todo_idx ON blocks (priority, id) WHERE status IN ('planned', 'leased')`
|
|
)
|
|
this.schema.raw(`CREATE INDEX blocks_leased_idx ON blocks (leased_at) WHERE status = 'leased'`)
|
|
this.schema.raw(
|
|
`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS block_id bigint REFERENCES blocks (id) ON DELETE SET NULL`
|
|
)
|
|
this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_block_idx ON ranges (block_id)`)
|
|
// The reset: every range planned again from its start (the corpus serves the stored ones). Priorities, capture,
|
|
// est_date, stored_at, plain_chunks and the measured posted_low/high stay.
|
|
this.schema.raw(`
|
|
UPDATE ranges SET status = 'planned', next_id = start_id, node_id = NULL, attempts = 0, spooled_chunks = 0,
|
|
spooled_at = NULL, leased_at = NULL, heartbeat_at = NULL, done_at = NULL, headers = 0, error = NULL,
|
|
rule_version = NULL
|
|
WHERE status <> 'planned' OR next_id <> start_id OR spooled_chunks <> 0 OR node_id IS NOT NULL`)
|
|
}
|
|
|
|
async down() {
|
|
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS block_id`)
|
|
this.schema.raw(`DROP TABLE IF EXISTS blocks`)
|
|
}
|
|
}
|