Partition files by age (id)

New rows went into free space scattered through the 9.6 GB heap on the HDD
(left behind by inspection's updates): one random HDD read per insert, and
seed batches took a median 13 s to store. The existing table becomes the
first partition as it is (a CHECK scan, no rewrite; indexes, its foreign
key and the triggers are taken over by the partitioned parent); new
partitions of 4M ids go in the same tablespace and stay in memory while
they fill.

- Every partition has a unique file_key index; FileStore.insert looks keys
  up in all partitions first (a key can't be unique across partitions).
- FilePartitions.ensure() (gc:scheduler) keeps two partitions ahead.
- Stats count files from the partitions; seed:node waits ~20 minutes for
  the API before giving a range up.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/FilePartitions.ts

@@ -0,0 +1,39 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Ids per partition (as the migration that partitioned files: …150). */
+export const FILE_PARTITION_SIZE = 4_000_000
+/** Ids kept covered beyond the newest file: two partitions. */
+const AHEAD = 2 * FILE_PARTITION_SIZE
+
+/**
+ * `files` is partitioned by id (CONCEPT.md D39). An insert with an id no
+ * partition covers fails, so gc:scheduler calls ensure() every sweep to keep
+ * two partitions ready ahead of the newest id. Each new partition gets its
+ * own unique index on file_key (FileStore.insert checks the older ones),
+ * in the parent's tablespace (the HDD on homelab).
+ */
+class FilePartitions {
+  async ensure(): Promise<string[]> {
+    const created: string[] = []
+    while (true) {
+      const { rows } = await db.rawQuery(
+        `SELECT (SELECT coalesce(max(id), 0) FROM files) AS max_id,
+                -- ''{0,1}, not ''?: knex reads ? as a binding.
+                max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
+         FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
+         WHERE i.inhparent = 'files'::regclass`
+      )
+      const maxId = Number(rows[0].max_id)
+      const upper = Number(rows[0].upper)
+      if (upper - maxId >= AHEAD) return created
+      const name = `files_p${upper}`
+      await db.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
+      await db.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
+      logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`)
+      created.push(name)
+    }
+  }
+}
+
+export default new FilePartitions()

app/services/FileStore.ts

@@ -39,15 +39,34 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
 class FileStore {
   /**
    * Inserts files, skipping those whose `file_key` exists. Returns the rows
-   * that were inserted (id and filename).
+   * that were inserted (id and filename). `files` is partitioned by id
+   * (CONCEPT.md D39) and each partition has its own unique file_key index:
+   * keys already in any partition are looked up first; two writers adding
+   * the same new key at once meet in the newest partition's index.
    */
   async insert(files: NewFile[], client: Pick<QueryClientContract, 'knexQuery'> = db): Promise<{ id: number; filename: string }[]> {
     if (files.length === 0) return []
-    const rows = files.map(({ segments, ...file }) => {
-      const sorted = [...segments].sort((a, b) => a.number - b.number)
-      return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
-    })
-    const inserted = await client.knexQuery().table('files').insert(rows).onConflict('file_key').ignore().returning(['id', 'filename'])
+    const keys = [...new Set(files.map((file) => file.file_key).filter((key): key is string => typeof key === 'string'))]
+    const known = new Set<string>()
+    for (let i = 0; i < keys.length; i += 5000) {
+      const found = await client.knexQuery().from('files').whereIn('file_key', keys.slice(i, i + 5000)).select('file_key')
+      for (const row of found) known.add(row.file_key)
+    }
+    const seen = new Set<string>()
+    const rows = files
+      .filter((file) => {
+        const key = file.file_key as string | null | undefined
+        if (!key) return true
+        if (known.has(key) || seen.has(key)) return false
+        seen.add(key)
+        return true
+      })
+      .map(({ segments, ...file }) => {
+        const sorted = [...segments].sort((a, b) => a.number - b.number)
+        return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
+      })
+    if (rows.length === 0) return []
+    const inserted = await client.knexQuery().table('files').insert(rows).onConflict().ignore().returning(['id', 'filename'])
     return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
   }
 }

app/services/SearchService.ts

@@ -390,7 +390,8 @@ class SearchService {
         SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
         FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
       SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
-             (SELECT reltuples FROM pg_class WHERE relname = 'files') AS files,
+             -- files is partitioned (D39): its partitions hold the counts.
+             (SELECT sum(c.reltuples) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass) AS files,
              (SELECT count(*) FROM sample) AS sampled,
              (SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
              -- Indexed data only: a spot listing's size is what the spot announces.