file_keys: fill in the background, use it once complete

Copying all existing keys in the migration read the files rows from the
HDD for over half an hour and held up the deploy. gc:scheduler now copies
them in slices (FILE_KEYS_FILL_SECONDS per sweep) and FileStore checks
every partition until the fill is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/FileKeys.ts

@@ -0,0 +1,60 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** files ids per slice: a few seconds of reads from the HDD. */
+const SLICE = 100_000
+
+/**
+ * The file_keys filter (LOG.md 2026-10-06 (7)): a 64-bit hash of every
+ * file_key. The insert trigger adds new keys; `fill` copies the keys of
+ * files from before, in id order and a slice at a time, and marks the
+ * filter complete when it reaches the files that existed when it started
+ * (everything after that has been added by the trigger, which was live
+ * first). FileStore uses the filter only once it is complete.
+ */
+class FileKeys {
+  private completeSince = 0
+
+  /** Whether every file's key is in file_keys; cached once true. */
+  async complete(): Promise<boolean> {
+    if (this.completeSince) return true
+    const row = await db.from('file_keys_fill').where('id', 1).first()
+    if (row?.complete) this.completeSince = Date.now()
+    return Boolean(row?.complete)
+  }
+
+  /** Forgets the cached state (tests). */
+  forget() {
+    this.completeSince = 0
+  }
+
+  /** Copies keys until `budgetMs` has passed or the fill is done; returns the files read and whether it's complete. */
+  async fill(budgetMs: number): Promise<{ read: number; complete: boolean; nextId: number; untilId: number }> {
+    const started = Date.now()
+    let state = await db.from('file_keys_fill').where('id', 1).first()
+    if (!state) return { read: 0, complete: false, nextId: 0, untilId: 0 }
+    if (state.until_id === null) {
+      // Everything inserted from here on goes through the trigger (live since the migration).
+      const [{ max }] = await db.from('files').max('id as max')
+      await db.from('file_keys_fill').where('id', 1).update({ until_id: Number(max ?? 0), updated_at: new Date() })
+      state = await db.from('file_keys_fill').where('id', 1).first()
+    }
+    let next = Number(state.next_id)
+    const until = Number(state.until_id)
+    let read = 0
+    while (next <= until && Date.now() - started < budgetMs) {
+      const to = Math.min(until, next + SLICE - 1)
+      const result = await db.rawQuery(
+        `INSERT INTO file_keys (key_hash)
+         SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE id BETWEEN ? AND ? AND file_key IS NOT NULL
+         ON CONFLICT DO NOTHING`,
+        [next, to]
+      )
+      read += Number(result.rowCount ?? 0)
+      next = to + 1
+      await db.from('file_keys_fill').where('id', 1).update({ next_id: next, complete: next > until, updated_at: new Date() })
+    }
+    return { read, complete: next > until, nextId: next, untilId: until }
+  }
+}
+
+export default new FileKeys()

app/services/FileStore.ts

@@ -1,4 +1,5 @@
 import db from '@adonisjs/lucid/services/db'
+import FileKeys from '#services/FileKeys'
 import type { QueryClientContract } from '@adonisjs/lucid/types/database'
 import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
 
@@ -44,8 +45,8 @@ class FileStore {
    * Inserts files, skipping those whose `file_key` exists. Returns the rows
    * that were inserted (id, filename, file_key). `files` is partitioned by id
    * (CONCEPT.md D39) and each partition has its own unique file_key index:
-   * file_keys (a 64-bit hash of every key, filled by the insert trigger) says
-   * which keys may exist, and only those are looked up in the partitions; two
+   * once file_keys (a 64-bit hash of every key) is complete, it says which keys
+   * may exist, and only those are looked up in the partitions; two
    * writers adding the same new key at once meet in the newest partition's index.
    */
   async insert(
@@ -55,8 +56,14 @@ class FileStore {
     if (files.length === 0) return []
     const keys = [...new Set(files.map((file) => file.file_key).filter((key): key is string => typeof key === 'string'))]
     const known = new Set<string>()
+    const filtered = await FileKeys.complete()
     for (let i = 0; i < keys.length; i += 5000) {
       const chunk = keys.slice(i, i + 5000)
+      if (!filtered) {
+        const found = await client.knexQuery().from('files').whereIn('file_key', chunk).select('file_key')
+        for (const row of found) known.add(row.file_key)
+        continue
+      }
       const { rows } = await client.rawQuery(
         `SELECT k.key FROM unnest(?::text[]) AS k(key)
          WHERE EXISTS (SELECT 1 FROM file_keys f WHERE f.key_hash = hashtextextended(k.key, 0))`,