import { BaseSchema } from '@adonisjs/lucid/schema' /** Ids per partition: 4M files is a few days of indexing, ~2 GB of heap (FilePartitions uses the same). */ const FILE_PARTITION_SIZE = 4_000_000 /** * `files` partitioned by id, i.e. by age (CONCEPT.md D39). New rows went into * free space scattered through the 9.6 GB heap on the HDD (left by * inspection's updates), one random read per insert; now they go into a * small recent partition that stays in memory. The existing table becomes * the first partition as it is: only a CHECK scan, no rewrite. * * Postgres can't enforce a unique file_key across partitions: every * partition has its own unique index, and FileStore.insert looks keys up in * the older ones. Run with the writers stopped (a few minutes). */ export default class extends BaseSchema { static disableTransactions = true async up() { const q = (sql: string) => this.db.rawQuery(sql) const { rows } = await q(`SELECT coalesce(max(id), 0) AS max FROM files`) // The first new partition starts at the next million. const boundary = (Math.floor(Number(rows[0].max) / 1_000_000) + 1) * 1_000_000 // The HDD tablespace on homelab (D32), the default elsewhere; new partitions inherit the parent's. const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`) const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : '' // The old table, renamed; its constraint and index names get a files_p0 prefix. await q(`ALTER TABLE files ADD CONSTRAINT files_p0_bound CHECK (id < ${boundary}) NOT VALID`) await q(`ALTER TABLE files VALIDATE CONSTRAINT files_p0_bound`) await q(`ALTER TABLE files RENAME TO files_p0`) await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_pkey TO files_p0_pkey`) await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_file_key_unique TO files_p0_file_key_unique`) await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_release_id_foreign TO files_p0_release_id_foreign`) for (const index of ['files_content_type_index', 'files_first_message_id_idx', 'files_name_idx', 'files_release_id_index']) { await q(`ALTER INDEX ${index} RENAME TO ${index.replace(/^files_/, 'files_p0_')}`) } // Recreated on the parent, which clones them to every partition. for (const trigger of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) { await q(`DROP TRIGGER ${trigger} ON files_p0`) } // The parent: same columns and defaults (the id sequence), partitioned by id, in the old table's tablespace. await q(`CREATE TABLE files (LIKE files_p0 INCLUDING DEFAULTS) PARTITION BY RANGE (id)${tablespace}`) await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`) await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`) await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`) await q(`CREATE INDEX files_content_type_index ON files (content_type)`) await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`) await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`) await q(`CREATE INDEX files_release_id_index ON files (release_id)`) // Matching indexes and the foreign key of files_p0 are taken over, not rebuilt; the CHECK spares the scan. await q(`ALTER TABLE files ATTACH PARTITION files_p0 FOR VALUES FROM (MINVALUE) TO (${boundary})`) await q(`ALTER TABLE files_p0 DROP CONSTRAINT files_p0_bound`) await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`) await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`) await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`) await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`) // binaries.file_id followed the rename to files_p0; point it at the parent (no scan needed). await q(`ALTER TABLE binaries DROP CONSTRAINT binaries_file_id_foreign`) await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`) // The next two partitions; FilePartitions.ensure() (gc:scheduler) keeps two ahead from now on. for (const start of [boundary, boundary + FILE_PARTITION_SIZE]) { const name = `files_p${start}` await q(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${start}) TO (${start + FILE_PARTITION_SIZE})`) await q(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`) } } async down() { throw new Error('Not reversible: merging partitions back means rewriting the files table.') } }