Partition files by age (id)

New rows went into free space scattered through the 9.6 GB heap on the HDD (left behind by inspection's updates): one random HDD read per insert, and seed batches took a median 13 s to store. The existing table becomes the first partition as it is (a CHECK scan, no rewrite; indexes, its foreign key and the triggers are taken over by the partitioned parent); new partitions of 4M ids go in the same tablespace and stay in memory while they fill. - Every partition has a unique file_key index; FileStore.insert looks keys up in all partitions first (a key can't be unique across partitions). - FilePartitions.ensure() (gc:scheduler) keeps two partitions ahead. - Stats count files from the partitions; seed:node waits ~20 minutes for the API before giving a range up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 15:57:51 +02:00
1 parent 41859636d3
commit f1773744c0
@@ -0,0 +1,77 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Ids per partition: 4M files is a few days of indexing, ~2 GB of heap (FilePartitions uses the same). */
+const FILE_PARTITION_SIZE = 4_000_000
+
+/**
+ * `files` partitioned by id, i.e. by age (CONCEPT.md D39). New rows went into
+ * free space scattered through the 9.6 GB heap on the HDD (left by
+ * inspection's updates), one random read per insert; now they go into a
+ * small recent partition that stays in memory. The existing table becomes
+ * the first partition as it is: only a CHECK scan, no rewrite.
+ *
+ * Postgres can't enforce a unique file_key across partitions: every
+ * partition has its own unique index, and FileStore.insert looks keys up in
+ * the older ones. Run with the writers stopped (a few minutes).
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    const { rows } = await q(`SELECT coalesce(max(id), 0) AS max FROM files`)
+    // The first new partition starts at the next million.
+    const boundary = (Math.floor(Number(rows[0].max) / 1_000_000) + 1) * 1_000_000
+    // The HDD tablespace on homelab (D32), the default elsewhere; new partitions inherit the parent's.
+    const { rows: ts } = await q(`SELECT t.spcname FROM pg_class c JOIN pg_tablespace t ON t.oid = c.reltablespace WHERE c.oid = 'files'::regclass`)
+    const tablespace = ts[0]?.spcname ? ` TABLESPACE ${ts[0].spcname}` : ''
+
+    // The old table, renamed; its constraint and index names get a files_p0 prefix.
+    await q(`ALTER TABLE files ADD CONSTRAINT files_p0_bound CHECK (id < ${boundary}) NOT VALID`)
+    await q(`ALTER TABLE files VALIDATE CONSTRAINT files_p0_bound`)
+    await q(`ALTER TABLE files RENAME TO files_p0`)
+    await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_pkey TO files_p0_pkey`)
+    await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_file_key_unique TO files_p0_file_key_unique`)
+    await q(`ALTER TABLE files_p0 RENAME CONSTRAINT files_release_id_foreign TO files_p0_release_id_foreign`)
+    for (const index of ['files_content_type_index', 'files_first_message_id_idx', 'files_name_idx', 'files_release_id_index']) {
+      await q(`ALTER INDEX ${index} RENAME TO ${index.replace(/^files_/, 'files_p0_')}`)
+    }
+    // Recreated on the parent, which clones them to every partition.
+    for (const trigger of ['files_release_extension', 'files_release_search', 'files_release_totals', 'files_zz_release_complete']) {
+      await q(`DROP TRIGGER ${trigger} ON files_p0`)
+    }
+
+    // The parent: same columns and defaults (the id sequence), partitioned by id, in the old table's tablespace.
+    await q(`CREATE TABLE files (LIKE files_p0 INCLUDING DEFAULTS) PARTITION BY RANGE (id)${tablespace}`)
+    await q(`ALTER SEQUENCE files_id_seq OWNED BY files.id`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_pkey PRIMARY KEY (id)`)
+    await q(`ALTER TABLE files ADD CONSTRAINT files_release_id_foreign FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE SET NULL`)
+    await q(`CREATE INDEX files_content_type_index ON files (content_type)`)
+    await q(`CREATE INDEX files_first_message_id_idx ON files (first_message_id)`)
+    await q(`CREATE INDEX files_name_idx ON files ((coalesce(real_name, filename::text)))`)
+    await q(`CREATE INDEX files_release_id_index ON files (release_id)`)
+    // Matching indexes and the foreign key of files_p0 are taken over, not rebuilt; the CHECK spares the scan.
+    await q(`ALTER TABLE files ATTACH PARTITION files_p0 FOR VALUES FROM (MINVALUE) TO (${boundary})`)
+    await q(`ALTER TABLE files_p0 DROP CONSTRAINT files_p0_bound`)
+
+    await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+    await q(`CREATE TRIGGER files_release_totals AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files FOR EACH ROW EXECUTE FUNCTION files_release_totals()`)
+    await q(`CREATE TRIGGER files_zz_release_complete AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_complete()`)
+
+    // binaries.file_id followed the rename to files_p0; point it at the parent (no scan needed).
+    await q(`ALTER TABLE binaries DROP CONSTRAINT binaries_file_id_foreign`)
+    await q(`ALTER TABLE binaries ADD CONSTRAINT binaries_file_id_foreign FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE SET NULL NOT VALID`)
+
+    // The next two partitions; FilePartitions.ensure() (gc:scheduler) keeps two ahead from now on.
+    for (const start of [boundary, boundary + FILE_PARTITION_SIZE]) {
+      const name = `files_p${start}`
+      await q(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${start}) TO (${start + FILE_PARTITION_SIZE})`)
+      await q(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
+    }
+  }
+
+  async down() {
+    throw new Error('Not reversible: merging partitions back means rewriting the files table.')
+  }
+}
@@ -209,6 +209,117 @@ export class FileSchema extends BaseModel {
  declare updatedAt: DateTime | null
 }

+export class FilesP0Schema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
+  $columns = FilesP0Schema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column()
+  declare date: bigint | number
+  @column()
+  declare fileKey: string | null
+  @column()
+  declare filename: string
+  @column()
+  declare firstMessageId: string | null
+  @column()
+  declare groups: any | null
+  @column({ isPrimary: true })
+  declare id: number
+  @column()
+  declare parts: number
+  @column()
+  declare poster: string
+  @column()
+  declare realName: string | null
+  @column()
+  declare releaseId: bigint | number | null
+  @column()
+  declare segments: Buffer
+  @column()
+  declare size: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
+export class FilesP1000000Schema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
+  $columns = FilesP1000000Schema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column()
+  declare date: bigint | number
+  @column()
+  declare fileKey: string | null
+  @column()
+  declare filename: string
+  @column()
+  declare firstMessageId: string | null
+  @column()
+  declare groups: any | null
+  @column({ isPrimary: true })
+  declare id: number
+  @column()
+  declare parts: number
+  @column()
+  declare poster: string
+  @column()
+  declare realName: string | null
+  @column()
+  declare releaseId: bigint | number | null
+  @column()
+  declare segments: Buffer
+  @column()
+  declare size: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
+export class FilesP5000000Schema extends BaseModel {
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
+  $columns = FilesP5000000Schema.$columns
+  @column()
+  declare archive: any | null
+  @column()
+  declare contentType: string | null
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime | null
+  @column()
+  declare date: bigint | number
+  @column()
+  declare fileKey: string | null
+  @column()
+  declare filename: string
+  @column()
+  declare firstMessageId: string | null
+  @column()
+  declare groups: any | null
+  @column({ isPrimary: true })
+  declare id: number
+  @column()
+  declare parts: number
+  @column()
+  declare poster: string
+  @column()
+  declare realName: string | null
+  @column()
+  declare releaseId: bigint | number | null
+  @column()
+  declare segments: Buffer
+  @column()
+  declare size: bigint | number | null
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime | null
+}
+
 export class GroupSchema extends BaseModel {
  static $columns = ['active', 'backfillNextId', 'backfillUntilId', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
  $columns = GroupSchema.$columns