From 9201e1200eaf8329bdf51180b952b71a904275c5 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 5 Oct 2026 16:46:53 +0200 Subject: [PATCH] files.id to 64 bits Seeding adds ~14M files a day; 32-bit ids run out at 2.1 billion in about five months. Postgres can't change a partition key's type, so the migration detaches the partitions, rewrites each as a plain table (a CHECK added first spares the attach scan), and attaches them to a new 64-bit parent with the same indexes, foreign keys and triggers. binaries.file_id and the id sequence become 64-bit too. Meant to run with the writers stopped and Postgres on wal_level=minimal (compose: PG_WAL_LEVEL, PG_MAX_WAL_SENDERS). Co-Authored-By: Claude Opus 5.5 --- .../migrations/1768620765160_files_bigint.ts | 89 +++++++++++++++++++ database/schema.ts | 10 +-- 2 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 database/migrations/1768620765160_files_bigint.ts diff --git a/database/migrations/1768620765160_files_bigint.ts b/database/migrations/1768620765160_files_bigint.ts new file mode 100644 index 0000000..8453f34 --- /dev/null +++ b/database/migrations/1768620765160_files_bigint.ts @@ -0,0 +1,89 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +const TRIGGERS = [ + `files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`, + `files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`, + `files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`, + `files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`, +] + +/** + * files.id (and binaries.file_id, the id sequence) to 64 bits (CONCEPT.md + * D39): seeding adds ~14M files a day, and 32 bits run out at 2.1 billion. + * Postgres can't change the type of a partition key, so the partitions are + * detached, rewritten one by one as plain tables (files_p0: ~100 GB with its + * segment lists, 1–2 hours on homelab's HDD) and attached to a new 64-bit + * parent. A CHECK per partition, added before its rewrite, spares the attach + * scan. Run with the writers stopped and Postgres on wal_level=minimal, or the + * rewrite also goes through the WAL (LOG.md 2026-10-05 (8)). + */ +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + const q = (sql: string) => this.db.rawQuery(sql) + const { rows: partitions } = await q( + `SELECT c.relname AS name, pg_get_expr(c.relpartbound, c.oid) AS bound + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname` + ) + const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`) + const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : '' + + // The sequence belongs to the old parent: keep it when that goes. + await q(`ALTER SEQUENCE files_id_seq OWNED BY NONE`) + await q(`ALTER SEQUENCE files_id_seq AS bigint`) + await q(`ALTER TABLE binaries DROP CONSTRAINT IF EXISTS binaries_file_id_foreign`) + await q(`ALTER TABLE binaries ALTER COLUMN file_id TYPE bigint`) + + for (const p of partitions) { + await q(`ALTER TABLE files DETACH PARTITION ${p.name}`) + // Triggers cloned from the parent may stay behind on a detached partition; the new parent clones them again. + for (const t of TRIGGERS) await q(`DROP TRIGGER IF EXISTS ${t.split(' ')[0]} ON ${p.name}`) + const [, from, to] = /FROM \((.+)\) TO \((.+)\)/.exec(p.bound.replaceAll("'", ''))! + const conditions = [from === 'MINVALUE' ? null : `id >= ${from}`, to === 'MAXVALUE' ? null : `id < ${to}`].filter(Boolean) + if (conditions.length > 0) await q(`ALTER TABLE ${p.name} ADD CONSTRAINT ${p.name}_bound CHECK (${conditions.join(' AND ')})`) + } + await q(`DROP TABLE files`) + + for (const p of partitions) { + await q(`ALTER TABLE ${p.name} ALTER COLUMN id TYPE bigint`) + } + + await q(` + CREATE TABLE files ( + id bigint NOT NULL DEFAULT nextval('files_id_seq'), + filename varchar(255) NOT NULL, + poster varchar(255) NOT NULL, + date bigint NOT NULL, + parts integer NOT NULL, + groups jsonb, + created_at timestamptz, + updated_at timestamptz, + file_key varchar(40), + release_id bigint, + archive jsonb, + real_name text, + size bigint, + content_type varchar(16), + segments bytea NOT NULL, + first_message_id text + ) PARTITION BY RANGE (id)${tablespace}`) + await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`) + await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`) + await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`) + await q(`CREATE INDEX files_content_type_index ON files (content_type)`) + await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`) + await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`) + await q(`CREATE INDEX files_release_id_index ON files (release_id)`) + for (const p of partitions) { + await q(`ALTER TABLE files ATTACH PARTITION ${p.name} ${p.bound}`) + await q(`ALTER TABLE ${p.name} DROP CONSTRAINT IF EXISTS ${p.name}_bound`) + } + for (const t of TRIGGERS) await q(`CREATE TRIGGER ${t}`) + await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`) + } + + async down() { + throw new Error('Not reversible: ids above 2^31 may exist.') + } +} diff --git a/database/schema.ts b/database/schema.ts index 2def622..e16a281 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -40,7 +40,7 @@ export class BinarySchema extends BaseModel { @column.dateTime() declare conflictedAt: DateTime | null @column() - declare fileId: number | null + declare fileId: bigint | number | null @column() declare fileKey: string @column.dateTime() @@ -192,7 +192,7 @@ export class FileSchema extends BaseModel { @column() declare groups: any | null @column({ isPrimary: true }) - declare id: number + declare id: bigint | number @column() declare parts: number @column() @@ -229,7 +229,7 @@ export class FilesP0Schema extends BaseModel { @column() declare groups: any | null @column({ isPrimary: true }) - declare id: number + declare id: bigint | number @column() declare parts: number @column() @@ -266,7 +266,7 @@ export class FilesP1000000Schema extends BaseModel { @column() declare groups: any | null @column({ isPrimary: true }) - declare id: number + declare id: bigint | number @column() declare parts: number @column() @@ -303,7 +303,7 @@ export class FilesP5000000Schema extends BaseModel { @column() declare groups: any | null @column({ isPrimary: true }) - declare id: number + declare id: bigint | number @column() declare parts: number @column()