Seed plans fill every uncovered gap, with priorities; cheaper ingest and file storing

- seed:plan plans every part of a group's history no range covers (it
  started below the lowest known range and left a.b.ath's 13 most recent
  months out); seed_ranges.priority orders leases (seed:plan --priority).
- binaries.part_count, kept by the parts insert in the same statement:
  claimCompleted compares it instead of counting parts every batch.
- file_keys: 64-bit hashes of all file keys; FileStore.insert asks it
  before probing every files partition.
- worker:file stores 100 completed binaries per job with set-based queries.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/binary_files.spec.ts

@@ -0,0 +1,68 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import IngestService, { type IngestSegment } from '#services/IngestService'
+import BinaryFileService from '#services/BinaryFileService'
+import QueueService from '#services/QueueService'
+import FileStore, { segmentsOf } from '#services/FileStore'
+import { fileKey } from '#lib/subject/index'
+
+function segment(name: string, number: number, total = 3): IngestSegment {
+  const poster = 'batch-test@example.invalid'
+  const subject = `"${name}" yEnc`
+  return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] }
+}
+
+const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first()
+
+test.group('Binaries to files', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.teardown(() => QueueService.closeAll())
+
+  test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => {
+    await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)])
+    // The same article again (a crosspost): not counted twice.
+    await IngestService.ingest([segment('a.bin', 2)])
+    assert.equal((await binary('a.bin')).part_count, 2)
+    assert.isNull((await binary('a.bin')).completed_at)
+    const completed = await IngestService.ingest([segment('a.bin', 3)])
+    assert.equal((await binary('a.bin')).part_count, 3)
+    assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey])
+  })
+
+  test('a binary from before the count is counted exactly on its next part', async ({ assert }) => {
+    await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)])
+    await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null })
+    const completed = await IngestService.ingest([segment('b.bin', 3)])
+    assert.equal((await binary('b.bin')).part_count, 3)
+    assert.lengthOf(completed, 1)
+  })
+
+  test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => {
+    await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)]))
+    await IngestService.ingest([segment('half.bin', 1)])
+    // known.bin is a file already (e.g. from an NZB), found by its first segment.
+    const now = new Date()
+    const [known] = await FileStore.insert([
+      { filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now },
+    ])
+    const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id))
+
+    const result = await BinaryFileService.store(ids)
+    assert.include(result, { saved: 1, existing: 1 })
+    assert.deepEqual(result.incomplete, [ids[2]])
+
+    const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first()
+    assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
+    assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id))
+    assert.equal(Number((await binary('known.bin')).file_id), known.id)
+    // Header parts are in the file's segment list now.
+    const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n')
+    assert.equal(Number(n), 0)
+
+    // The retry: done ones are skipped, the incomplete one again incomplete.
+    const again = await BinaryFileService.store(ids)
+    assert.include(again, { saved: 0, skipped: 2 })
+    assert.deepEqual(again.incomplete, [ids[2]])
+  })
+})

tests/functional/seed_api.spec.ts

@@ -134,6 +134,16 @@ test.group('Seed API', (group) => {
     }
   })
 
+  test('ranges of a higher priority are leased first', async ({ client, assert }) => {
+    const token = await SeedOrchestrator.issueToken('priority-node')
+    await db.table('seed_ranges').insert([
+      { group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
+      { group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101, priority: 1 },
+    ])
+    const leased = await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })
+    assert.include(leased.body().range, { start: 101, end: 200 })
+  })
+
   test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
     await QueueService.collectionQueue.drain()
     await QueueService.archiveWalkQueue.drain()

tests/unit/seed_plan.spec.ts

@@ -0,0 +1,16 @@
+import { test } from '@japa/runner'
+import { cut, uncovered } from '#lib/seed/plan'
+
+test.group('seed plan', () => {
+  test('plans only what earlier ranges leave uncovered', ({ assert }) => {
+    // a.b.ath on 2026-10-06: test ranges around 49.8B and a plan below them; the backfill ends at 58.4B.
+    assert.deepEqual(uncovered(1, 100, [[40, 50], [10, 20], [45, 60]]), [[1, 9], [21, 39], [61, 100]])
+    assert.deepEqual(uncovered(1, 100, []), [[1, 100]])
+    assert.deepEqual(uncovered(1, 100, [[0, 200]]), [])
+    assert.deepEqual(uncovered(10, 20, [[1, 5], [25, 30]]), [[10, 20]])
+  })
+
+  test('cuts gaps into ranges, newest first', ({ assert }) => {
+    assert.deepEqual(cut([[1, 9], [61, 100]], 25), [[76, 100], [61, 75], [1, 9]])
+  })
+})