Seed plans fill every uncovered gap, with priorities; cheaper ingest and file storing

- seed:plan plans every part of a group's history no range covers (it
  started below the lowest known range and left a.b.ath's 13 most recent
  months out); seed_ranges.priority orders leases (seed:plan --priority).
- binaries.part_count, kept by the parts insert in the same statement:
  claimCompleted compares it instead of counting parts every batch.
- file_keys: 64-bit hashes of all file keys; FileStore.insert asks it
  before probing every files partition.
- worker:file stores 100 completed binaries per job with set-based queries.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/BinaryFileService.ts

@@ -0,0 +1,141 @@
+import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import QueueService from '#services/QueueService'
+import ContentService from '#services/ContentService'
+
+export interface BinaryFilesResult {
+  /** New files. */
+  saved: number
+  /** Binaries whose file already existed (same first segment or file_key). */
+  existing: number
+  /** Conflicted, gone or already linked: nothing to do. */
+  skipped: number
+  /** Not all parts stored yet (the job is retried for these). */
+  incomplete: number[]
+  inspections: number
+}
+
+/**
+ * Turns completed binaries into files (worker:file), a batch at a time
+ * (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and
+ * files that exist already, one INSERT for the new files (so the files
+ * statement trigger updates each release once), and one UPDATE and DELETE
+ * for the binaries' links and copied parts. It was one job of ~8 queries per
+ * binary, ~700 a minute.
+ */
+class BinaryFileService {
+  async store(binaryIds: number[]): Promise<BinaryFilesResult> {
+    const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 }
+    const ids = [...new Set(binaryIds)]
+    const binaries = await db
+      .from('binaries')
+      .whereIn('id', ids)
+      .whereNull('conflicted_at')
+      .whereNull('file_id')
+      .select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts')
+    result.skipped = ids.length - binaries.length
+    if (binaries.length === 0) return result
+
+    const partsOf = new Map<number, { number: number; id: string; size: number }[]>()
+    const parts = await db
+      .from('parts')
+      .whereIn('binary_id', binaries.map((b) => b.id))
+      .orderBy([{ column: 'binary_id' }, { column: 'number' }])
+      .select('binary_id', 'number', 'message_id', 'bytes')
+    for (const part of parts) {
+      const list = partsOf.get(Number(part.binary_id)) ?? []
+      list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) })
+      partsOf.set(Number(part.binary_id), list)
+    }
+    const complete = binaries.filter((b) => {
+      const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts
+      if (!ok) result.incomplete.push(Number(b.id))
+      return ok
+    })
+    if (complete.length === 0) return result
+
+    // Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list).
+    const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id)
+    const byFirst = new Map<string, number>()
+    for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) {
+      byFirst.set(row.first_message_id, Number(row.id))
+    }
+    const fileIdOf = new Map<number, number>()
+    const fresh = complete.filter((b) => {
+      const existing = byFirst.get(partsOf.get(Number(b.id))![0].id)
+      if (existing) fileIdOf.set(Number(b.id), existing)
+      return !existing
+    })
+
+    const now = new Date()
+    // file_key is unique: a second completion of the same upload inserts nothing.
+    const inserted = await FileStore.insert(
+      fresh.map((b) => {
+        const segments = partsOf.get(Number(b.id))!
+        return {
+          file_key: b.file_key,
+          filename: b.name,
+          poster: b.poster,
+          date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(),
+          parts: segments.length,
+          segments,
+          groups: JSON.stringify(b.groups),
+          release_id: b.release_id,
+          real_name: b.real_name,
+          size: segments.reduce((sum, s) => sum + s.size, 0),
+          created_at: now,
+          updated_at: now,
+        }
+      })
+    )
+    const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id]))
+    const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key)
+    const byKey = new Map<string, number>()
+    if (missing.length > 0) {
+      for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id))
+    }
+    for (const b of fresh) {
+      const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
+      if (id) fileIdOf.set(Number(b.id), id)
+    }
+    result.saved = inserted.length
+    result.existing = fileIdOf.size - inserted.length
+
+    // Links, locked in file_key order like ingest's upsert, so the two can't deadlock.
+    const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key))
+    if (linked.length > 0) {
+      await db.transaction(async (trx) => {
+        await trx.rawQuery(
+          `SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
+          [linked.map((b) => b.id)]
+        )
+        await trx.rawQuery(
+          `UPDATE binaries b SET file_id = v.file_id FROM unnest(?::bigint[], ?::bigint[]) AS v(id, file_id) WHERE b.id = v.id`,
+          [linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!)]
+        )
+      })
+      // Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
+      await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)])
+    }
+
+    // As for one file before: an SFV posted earlier may list them (D34), then inspection.
+    const fresher = new Map(fresh.map((b) => [b.file_key, b]))
+    const added = inserted.flatMap((row) => {
+      const b = row.file_key ? fresher.get(row.file_key) : undefined
+      return b
+        ? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }]
+        : []
+    })
+    if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file))
+    // By the posted name, as before: inspection skips later volumes and readable media names anyway.
+    const jobs = added
+      .filter((file) => ContentService.isCandidate(file.posted))
+      .map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
+    if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+    result.inspections = jobs.length
+    return result
+  }
+}
+
+export default new BinaryFileService()