Seed plans fill every uncovered gap, with priorities; cheaper ingest and file storing

- seed:plan plans every part of a group's history no range covers (it
  started below the lowest known range and left a.b.ath's 13 most recent
  months out); seed_ranges.priority orders leases (seed:plan --priority).
- binaries.part_count, kept by the parts insert in the same statement:
  claimCompleted compares it instead of counting parts every batch.
- file_keys: 64-bit hashes of all file keys; FileStore.insert asks it
  before probing every files partition.
- worker:file stores 100 completed binaries per job with set-based queries.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

database/migrations/1768620765200_seed_range_priority.ts

@@ -0,0 +1,14 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** Seed ranges of a higher priority are leased first (`seed:plan --priority`), e.g. a gap in recent history. */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('seed_ranges', (table) => {
+      table.smallint('priority').notNullable().defaultTo(0)
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('seed_ranges', (table) => table.dropColumn('priority'))
+  }
+}

database/migrations/1768620765210_binary_part_count.ts

@@ -0,0 +1,21 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Stored parts per binary, kept by IngestService as it inserts parts
+ * (LOG.md 2026-10-06 (7)). claimCompleted counted every binary's parts in
+ * every batch (15% of Postgres' active time); a binary of thousands of
+ * parts that gets a few per batch was counted thousands of times. NULL:
+ * not counted yet (binaries from before), counted exactly on the next new
+ * part. Only parts of binaries without a file are deleted, so it stays exact
+ * until a binary completes.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`SET LOCAL lock_timeout = '2min'`)
+    await this.db.rawQuery(`ALTER TABLE binaries ADD COLUMN IF NOT EXISTS part_count integer`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE binaries DROP COLUMN IF EXISTS part_count`)
+  }
+}

database/migrations/1768620765220_file_keys.ts

@@ -0,0 +1,68 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)).
+ * file_key is unique per files partition only (D39), so FileStore.insert
+ * looked every key up in every partition: 7 partitions and growing, 35,000
+ * random index reads for a batch of 5,000 files. Now it asks file_keys
+ * first, and only keys whose hash is there (a duplicate, or rarely a
+ * collision) are looked up in files.
+ *
+ * Filled by the files insert statement trigger; no transaction, so that the
+ * trigger is live before the existing keys are copied and no file inserted
+ * meanwhile is missed (a missed key would let a duplicate file in).
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`)
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
+        BEGIN
+          WITH added AS (
+            SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
+                   array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
+                   array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
+            FROM (
+              SELECT release_id, size, name_extension(nm) AS e,
+                     CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
+              FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
+            ) x
+            GROUP BY release_id
+          )
+          UPDATE releases r SET
+            file_count = r.file_count + a.n,
+            size = r.size + a.bytes,
+            extensions = CASE
+              WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
+              ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
+            search_files = CASE
+              WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
+              ELSE nullif(left(concat_ws(' ', r.search_files, (
+                SELECT string_agg(w, ' ') FROM unnest(a.words) w
+                WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
+          FROM added a
+          WHERE r.id = a.release_id;
+
+          INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
+          INSERT INTO file_keys (key_hash)
+            SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
+            ON CONFLICT DO NOTHING;
+          RETURN NULL;
+        END $$`)
+    // The keys from before; read from the file_key indexes (NVMe), not the files heap (HDD).
+    await q(`SET enable_seqscan = off`)
+    await q(`
+      INSERT INTO file_keys (key_hash)
+      SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE file_key IS NOT NULL
+      ON CONFLICT DO NOTHING`)
+    await q(`RESET enable_seqscan`)
+    await q(`ANALYZE file_keys`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`)
+  }
+}

database/schema.ts

@@ -33,7 +33,7 @@ export class AuthAccessTokenSchema extends BaseModel {
 }
 
 export class BinarySchema extends BaseModel {
-  static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
+  static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'partCount', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const
   $columns = BinarySchema.$columns
   @column.dateTime()
   declare completedAt: DateTime | null
@@ -57,6 +57,8 @@ export class BinarySchema extends BaseModel {
   declare name: string
   @column()
   declare parentBinaryId: bigint | number | null
+  @column()
+  declare partCount: number | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()
@@ -172,6 +174,13 @@ export class FetchRangeSchema extends BaseModel {
   declare updatedAt: DateTime
 }
 
+export class FileKeySchema extends BaseModel {
+  static $columns = ['keyHash'] as const
+  $columns = FileKeySchema.$columns
+  @column({ isPrimary: true })
+  declare keyHash: bigint | number
+}
+
 export class FileSchema extends BaseModel {
   static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
   $columns = FileSchema.$columns
@@ -486,7 +495,7 @@ export class SeedNodeSchema extends BaseModel {
 }
 
 export class SeedRangeSchema extends BaseModel {
-  static $columns = ['attempts', 'createdAt', 'doneAt', 'endId', 'error', 'files', 'fragments', 'groupName', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'startId', 'status'] as const
+  static $columns = ['attempts', 'createdAt', 'doneAt', 'endId', 'error', 'files', 'fragments', 'groupName', 'headers', 'heartbeatAt', 'id', 'leasedAt', 'nextId', 'nodeId', 'priority', 'startId', 'status'] as const
   $columns = SeedRangeSchema.$columns
   @column()
   declare attempts: number
@@ -517,6 +526,8 @@ export class SeedRangeSchema extends BaseModel {
   @column()
   declare nodeId: number | null
   @column()
+  declare priority: number
+  @column()
   declare startId: bigint | number
   @column()
   declare status: string