Seed orchestrator: remote nodes lease ranges and post results over HTTPS

- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with
  lease, resume point and counts); migration …140.
- SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a
  lease without a batch for 15 min passes to the next node and resumes at
  next_id), batch (stores files and fragments, moves next_id), finish,
  release, plan (from below live indexing / the backfill / earlier plans
  down to N days back; takes over the group's backfill), status.
- Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release,
  status) behind a node-token middleware; JSON bodies up to 64 MB after
  gzip.
- seed:node runs a node: no database or Redis, outgoing HTTPS only, N
  connections, gzip batches, retries for network errors and 5xx, drops a
  range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose
  file that builds the image from this repository.
- seed:token, seed:plan, seed:status for the operator. Header parsing moved
  to lib/ingest and the range loop to lib/seed/runner, shared by seed:range
  and seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/SeedService.ts

@@ -1,10 +1,11 @@
 import db from '@adonisjs/lucid/services/db'
-import type { AggregatedFile } from '#lib/aggregate/index'
-import IngestService, { type IngestSegment } from '#services/IngestService'
+import IngestService from '#services/IngestService'
 import FileStore from '#services/FileStore'
 import SfvService from '#services/SfvService'
 import ContentService from '#services/ContentService'
 import QueueService from '#services/QueueService'
+import StagingService from '#services/StagingService'
+import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
 
 export interface SeedStoreResult {
   files: number
@@ -15,49 +16,42 @@ export interface SeedStoreResult {
 }
 
 /**
- * Stores files an edge node aggregated from headers (CONCEPT.md D37): their
- * releases, the files with their segment lists, and one completed binary per
- * file, as worker:file would have left it, so par2 naming, inspection and GC
- * treat them like any other file. A batch is a few statements, not one per
- * article.
+ * Stores what seeding nodes found (CONCEPT.md D37): per complete file its
+ * release, the file with its segment list, and one completed binary, as
+ * worker:file would have left it, so par2 naming, inspection and GC treat
+ * it like any other file; fragments take the regular ingest path. A batch
+ * is a few statements, not one per article.
  */
 class SeedService {
-  async store(files: AggregatedFile<IngestSegment>[]): Promise<SeedStoreResult> {
+  async store(batch: SeedFile[]): Promise<SeedStoreResult> {
     const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0 }
+    // A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
+    const files = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
     if (files.length === 0) return result
     const knex = db.connection().getWriteClient()
     const now = new Date()
+    const postedOf = (file: SeedFile) => (file.postedAt ? new Date(file.postedAt) : undefined)
 
-    // A file's identity and release come from its first segment, its date from the earliest.
-    const firstOf = (file: AggregatedFile<IngestSegment>) => file.segments[0]
-    const postedOf = (file: AggregatedFile<IngestSegment>) => {
-      const dates = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
-      return dates.length > 0 ? new Date(Math.min(...dates)) : undefined
-    }
     const releaseIds = await IngestService.upsertReleases(
       knex,
-      files.map((file) => ({ ...firstOf(file), postedAt: postedOf(file) })),
+      files.map((file) => ({ release: file.release, poster: file.poster, groups: file.groups, postedAt: postedOf(file) })),
       now
     )
     result.releases = releaseIds.size
 
-    const rows = files.map((file) => {
-      const first = firstOf(file)
-      const postedAt = postedOf(file)
-      return {
-        file_key: file.fileKey,
-        filename: first.name,
-        poster: first.poster,
-        date: (postedAt ?? now).getTime(),
-        parts: file.segments.length,
-        segments: file.segments.map((s) => ({ number: s.number, id: s.messageId, size: s.bytes })),
-        groups: JSON.stringify(first.groups),
-        release_id: first.release ? (releaseIds.get(first.release.key) ?? null) : null,
-        size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
-        created_at: now,
-        updated_at: now,
-      }
-    })
+    const rows = files.map((file) => ({
+      file_key: file.fileKey,
+      filename: file.name,
+      poster: file.poster,
+      date: (postedOf(file) ?? now).getTime(),
+      parts: file.segments.length,
+      segments: file.segments.map(([number, id, size]) => ({ number, id, size })),
+      groups: JSON.stringify(file.groups),
+      release_id: file.release ? (releaseIds.get(file.release.key) ?? null) : null,
+      size: file.segments.reduce((sum, [, , size]) => sum + size, 0),
+      created_at: now,
+      updated_at: now,
+    }))
     const inserted = await FileStore.insert(rows)
     result.files = inserted.length
     result.existing = rows.length - inserted.length
@@ -67,19 +61,19 @@ class SeedService {
       fileIds.set(row.file_key, Number(row.id))
     }
     // The completed binary worker:file would have left behind; an existing one just learns its file.
-    const binaries = rows.map((row, i) => ({
-      file_key: row.file_key,
-      poster: row.poster,
-      subject: firstOf(files[i]).subject,
-      name: row.filename,
-      total_parts: firstOf(files[i]).totalParts,
-      groups: row.groups,
-      posted_at: postedOf(files[i]) ?? null,
+    const binaries = files.map((file, i) => ({
+      file_key: file.fileKey,
+      poster: file.poster,
+      subject: file.subject,
+      name: file.name,
+      total_parts: file.totalParts,
+      groups: rows[i].groups,
+      posted_at: postedOf(file) ?? null,
       first_seen_at: now,
       last_seen_at: now,
       completed_at: now,
-      release_id: row.release_id,
-      file_id: fileIds.get(row.file_key) ?? null,
+      release_id: rows[i].release_id,
+      file_id: fileIds.get(file.fileKey) ?? null,
     }))
     binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
     await knex('binaries')
@@ -101,6 +95,16 @@ class SeedService {
     result.inspections = candidates.length
     return result
   }
+
+  /** Segments of files that didn't complete on a node: staging (D35) for lone ones, binaries and parts for the rest. */
+  async storeFragments(group: string, fragments: SeedFragment[]): Promise<void> {
+    if (fragments.length === 0) return
+    const segments = fragments.map(fromFragment)
+    const { ingest } = StagingService.enabled(group)
+      ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(group) })
+      : { ingest: segments }
+    await IngestService.ingest(ingest)
+  }
 }
 
 export default new SeedService()