Seed orchestrator: remote nodes lease ranges and post results over HTTPS

- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with
  lease, resume point and counts); migration …140.
- SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a
  lease without a batch for 15 min passes to the next node and resumes at
  next_id), batch (stores files and fragments, moves next_id), finish,
  release, plan (from below live indexing / the backfill / earlier plans
  down to N days back; takes over the group's backfill), status.
- Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release,
  status) behind a node-token middleware; JSON bodies up to 64 MB after
  gzip.
- seed:node runs a node: no database or Redis, outgoing HTTPS only, N
  connections, gzip batches, retries for network errors and 5xx, drops a
  range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose
  file that builds the image from this repository.
- seed:token, seed:plan, seed:status for the operator. Header parsing moved
  to lib/ingest and the range loop to lib/seed/runner, shared by seed:range
  and seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/IngestService.ts

@@ -4,43 +4,10 @@ import type { QueryClientContract } from '@adonisjs/lucid/types/database'
 import type { Knex } from 'knex'
 import QueueService from '#services/QueueService'
 import NameService from '#services/NameService'
-import { parseXrefGroups } from '#lib/nntp/xref'
-import { fileKey, parseSubject, weakFileKey } from '#lib/subject/index'
-import { releaseIdentity, type ReleaseIdentity } from '#lib/release/index'
-import { inferTotalParts, type YencMeta } from '#lib/yenc/index'
+import type { IngestSegment } from '#lib/ingest/segment'
 
-/** One segment, with the identity of the file it belongs to already resolved. */
-export interface IngestSegment {
-  fileKey: string
-  poster: string
-  /** Subject without segment counter. */
-  subject: string
-  name: string
-  totalParts: number
-  number: number
-  /** Without angle brackets. */
-  messageId: string
-  bytes: number
-  articleNumber?: bigint | number
-  postedAt?: Date
-  groups: string[]
-  /**
-   * The identity comes from the yEnc name only (obfuscated posts). Such names
-   * can be shared by several equal-sized files of one post, so a second
-   * article for the same part number marks the binary as conflicted.
-   */
-  weakIdentity?: boolean
-  /** Release (download) this file belongs to, when it can be determined. */
-  release?: ReleaseIdentity | null
-  /** Exact file size from =ybegin (body-derived segments only). */
-  size?: number
-  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
-  pcrc32?: string
-  partSize?: number
-  fileCrc32?: string
-  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
-  md5_16k?: string
-}
+// Header parsing is pure and lives in lib, so seeding nodes need no database (CONCEPT.md D37).
+export { segmentFromHeader, segmentFromYenc, type IngestSegment, type OverviewHeader } from '#lib/ingest/segment'
 
 export interface CompletedBinary {
   id: number
@@ -134,74 +101,6 @@ function validDate(date?: Date) {
   return date && !Number.isNaN(date.getTime()) ? date : null
 }
 
-/** An XOVER overview entry as returned by nntp-js. */
-export type OverviewHeader = Record<string, string>
-
-function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
-  const xrefGroups = parseXrefGroups(header.xref)
-  return {
-    poster: header.from ?? '',
-    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
-    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
-    articleNumber,
-    postedAt: header.date ? new Date(header.date) : undefined,
-    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
-  }
-}
-
-/** Builds a segment from the subject line; null when the subject is obfuscated. */
-export function segmentFromHeader(
-  header: OverviewHeader,
-  groupName: string,
-  articleNumber?: bigint | number
-): IngestSegment | null {
-  const parsed = header.subject ? parseSubject(header.subject) : null
-  if (!parsed || !header['message-id']) return null
-  const base = common(header, groupName, articleNumber)
-  return {
-    ...base,
-    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
-    subject: parsed.base,
-    name: parsed.name,
-    totalParts: parsed.totalSegments,
-    number: parsed.segment,
-    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
-  }
-}
-
-/**
- * Builds a segment for an obfuscated post from the yEnc header in its body.
- * Subject and poster can be random per article, so the identity comes from
- * the yEnc header only (see weakFileKey).
- */
-export function segmentFromYenc(
-  header: OverviewHeader,
-  groupName: string,
-  meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
-  articleNumber?: bigint | number
-): IngestSegment | null {
-  const base = common(header, groupName, articleNumber)
-  const totalParts = inferTotalParts(meta)
-  const number = meta.begin.part ?? 1
-  if (!base.messageId || number < 1 || number > totalParts) return null
-  const subject = `"${meta.begin.name}" yEnc`
-  return {
-    ...base,
-    fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
-    subject,
-    name: meta.begin.name,
-    totalParts,
-    number,
-    weakIdentity: true,
-    release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
-    size: meta.begin.size,
-    // Single-part posts only carry crc32, which is then also the part's CRC.
-    pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
-    partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
-    fileCrc32: meta.end?.crc32,
-  }
-}
-
 /**
  * Stores segments in `binaries`/`parts` and queues every binary that became
  * complete exactly once. Idempotent: ingesting the same segments again is a

app/services/SeedOrchestrator.ts

@@ -0,0 +1,188 @@
+import { createHash, randomBytes } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import SeedService from '#services/SeedService'
+import { articleAtDate } from '#lib/nntp/date_search'
+import { normalizeSeedFile, type SeedBatch } from '#lib/seed/wire'
+
+/** A lease without a batch for this long is given to another node; the range resumes at next_id. */
+export const LEASE_TTL_SECONDS = 15 * 60
+/** Leases per range before it counts as failed (a range that keeps breaking nodes). */
+const MAX_ATTEMPTS = 5
+/** Articles per XOVER probe when looking for an article by date. */
+const PROBE = 50
+
+export interface SeedLease {
+  id: number
+  group: string
+  /** Resume point: first article not reported yet. */
+  start: number
+  end: number
+}
+
+export class LeaseLost extends Error {}
+
+const hash = (token: string) => createHash('sha256').update(token).digest('hex')
+
+/**
+ * The seed orchestrator (CONCEPT.md D38): seeding nodes lease article
+ * ranges over HTTPS, report what they aggregated in batches (D37), and
+ * finish or give back the range. Nodes only make outgoing requests; a lease
+ * that goes quiet for LEASE_TTL_SECONDS passes to the next node that asks.
+ */
+class SeedOrchestrator {
+  /** Creates a node, or gives an existing one a new token. Returns the token (stored hashed only). */
+  async issueToken(name: string): Promise<string> {
+    const token = `sn_${randomBytes(24).toString('base64url')}`
+    await db
+      .table('seed_nodes')
+      .insert({ name, token_hash: hash(token), disabled: false })
+      .onConflict('name')
+      .merge({ token_hash: hash(token), disabled: false })
+    return token
+  }
+
+  async nodeForToken(token: string): Promise<{ id: number; name: string } | null> {
+    const node = await db.from('seed_nodes').where('token_hash', hash(token)).where('disabled', false).first()
+    if (!node) return null
+    if (!node.last_seen_at || Date.now() - new Date(node.last_seen_at).getTime() > 60_000) {
+      await db.from('seed_nodes').where('id', node.id).update({ last_seen_at: new Date() })
+    }
+    return { id: Number(node.id), name: node.name }
+  }
+
+  /** The next range for this node, oldest plan first; a quiet lease is taken over. */
+  async lease(nodeId: number): Promise<SeedLease | null> {
+    // Ranges that broke node after node stop being handed out.
+    await db.rawQuery(
+      `UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often')
+       WHERE status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?) AND attempts >= ?`,
+      [LEASE_TTL_SECONDS, MAX_ATTEMPTS]
+    )
+    const { rows } = await db.rawQuery(
+      `UPDATE seed_ranges SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1
+       WHERE id = (
+         SELECT id FROM seed_ranges
+         WHERE status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?))
+         ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED)
+       RETURNING id, group_name, next_id, end_id`,
+      [nodeId, LEASE_TTL_SECONDS]
+    )
+    const row = rows[0]
+    return row ? { id: Number(row.id), group: row.group_name, start: Number(row.next_id), end: Number(row.end_id) } : null
+  }
+
+  /** Stores a node's batch and moves the range's resume point; LeaseLost when the range is no longer this node's. */
+  async batch(nodeId: number, rangeId: number, batch: SeedBatch) {
+    const range = await this.leased(nodeId, rangeId)
+    const stored = await SeedService.store(batch.files.map(normalizeSeedFile))
+    await SeedService.storeFragments(range.group_name, batch.fragments)
+    await db.rawQuery(
+      `UPDATE seed_ranges SET next_id = greatest(next_id, least(?, end_id + 1)), heartbeat_at = now(),
+         headers = headers + ?, files = files + ?, fragments = fragments + ?
+       WHERE id = ?`,
+      [batch.next, batch.headers, batch.files.length, batch.fragments.length, rangeId]
+    )
+    return stored
+  }
+
+  async finish(nodeId: number, rangeId: number) {
+    await this.leased(nodeId, rangeId)
+    await db.from('seed_ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date(), error: null })
+  }
+
+  /** A node gives a range back (it failed, or is shutting down); it's queued again unless it failed too often. */
+  async release(nodeId: number, rangeId: number, error?: string) {
+    const range = await this.leased(nodeId, rangeId)
+    await db
+      .from('seed_ranges')
+      .where('id', rangeId)
+      .update({ status: range.attempts >= MAX_ATTEMPTS ? 'failed' : 'queued', node_id: null, error: error?.slice(0, 2000) ?? null })
+  }
+
+  private async leased(nodeId: number, rangeId: number) {
+    const range = await db.from('seed_ranges').where('id', rangeId).first()
+    if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
+    return range
+  }
+
+  /**
+   * Plans ranges for groups: from below what live indexing and the backfill
+   * cover (or below earlier seed ranges) down to `days` back, or to the
+   * server's oldest article. A group's backfill stops where its seed plan
+   * begins. Ranges are interleaved across groups, newest first.
+   */
+  async plan(groups: string[], options: { days?: number; size: number }): Promise<{ group: string; from: number; to: number; ranges: number }[]> {
+    const perGroup: { group: string; ranges: [number, number][] }[] = []
+    const summary: { group: string; from: number; to: number; ranges: number }[] = []
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      for (const name of groups) {
+        const group = await db.from('groups').where('name', name).first()
+        if (!group) throw new Error(`Unknown group ${name}`)
+        const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, name), `GROUP ${name}`)
+        const dateOf = async (n: bigint) => {
+          const { overviews } = await NntpService.timed<any>(conn.xover(n, n + BigInt(PROBE)), `XOVER ${name} ${n}`)
+          const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
+          const date = header ? Date.parse(header.date) : Number.NaN
+          return Number.isNaN(date) ? null : date
+        }
+        // Top: where live indexing began, or the backfill's position, or below earlier seed ranges.
+        let top = Number(await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf))
+        const backfilling = group.backfill_next_id !== null && group.backfill_until_id !== null && Number(group.backfill_next_id) > Number(group.backfill_until_id)
+        if (backfilling) top = Math.min(top, Number(group.backfill_next_id))
+        const seeded = await db.from('seed_ranges').where('group_name', name).min('start_id as low').first()
+        if (seeded?.low !== null && seeded?.low !== undefined) top = Math.min(top, Number(seeded.low))
+        const bottom = options.days
+          ? Number(await articleAtDate(first, last, Date.now() - options.days * 24 * 60 * 60 * 1000, dateOf))
+          : Number(first)
+
+        const ranges: [number, number][] = []
+        for (let end = top - 1; end >= bottom; end -= options.size) ranges.push([Math.max(bottom, end - options.size + 1), end])
+        perGroup.push({ group: name, ranges })
+        summary.push({ group: name, from: bottom, to: top - 1, ranges: ranges.length })
+        // The seed plan takes over the rest of this group's backfill.
+        if (backfilling && ranges.length > 0) {
+          await db.from('groups').where('id', group.id).update({ backfill_until_id: group.backfill_next_id })
+        }
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+
+    // Round-robin over groups, each newest first, so every group fills in from the present backwards.
+    const rows: Record<string, unknown>[] = []
+    for (let i = 0; perGroup.some((g) => i < g.ranges.length); i++) {
+      for (const g of perGroup) {
+        const range = g.ranges[i]
+        if (range) rows.push({ group_name: g.group, start_id: range[0], end_id: range[1], next_id: range[0] })
+      }
+    }
+    for (let i = 0; i < rows.length; i += 5000) await db.table('seed_ranges').multiInsert(rows.slice(i, i + 5000))
+    return summary
+  }
+
+  async status() {
+    const { rows: ranges } = await db.rawQuery(
+      `SELECT status, count(*)::int AS ranges, sum(end_id - start_id + 1)::bigint AS articles,
+              sum(next_id - start_id)::bigint AS reached, sum(headers)::bigint AS headers, sum(files)::bigint AS files
+       FROM seed_ranges GROUP BY status ORDER BY status`
+    )
+    const { rows: nodes } = await db.rawQuery(
+      `SELECT n.name, n.last_seen_at, n.disabled,
+              (SELECT count(*)::int FROM seed_ranges r WHERE r.node_id = n.id AND r.status = 'leased') AS leases,
+              (SELECT coalesce(sum(headers), 0)::bigint FROM seed_ranges r WHERE r.node_id = n.id) AS headers
+       FROM seed_nodes n ORDER BY n.name`
+    )
+    const { rows: recent } = await db.rawQuery(
+      `SELECT coalesce(sum(headers), 0)::bigint AS headers FROM seed_ranges WHERE done_at > now() - interval '1 hour'`
+    )
+    return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers) }
+  }
+}
+
+export default new SeedOrchestrator()

app/services/SeedService.ts

@@ -1,10 +1,11 @@
 import db from '@adonisjs/lucid/services/db'
-import type { AggregatedFile } from '#lib/aggregate/index'
-import IngestService, { type IngestSegment } from '#services/IngestService'
+import IngestService from '#services/IngestService'
 import FileStore from '#services/FileStore'
 import SfvService from '#services/SfvService'
 import ContentService from '#services/ContentService'
 import QueueService from '#services/QueueService'
+import StagingService from '#services/StagingService'
+import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
 
 export interface SeedStoreResult {
   files: number
@@ -15,49 +16,42 @@ export interface SeedStoreResult {
 }
 
 /**
- * Stores files an edge node aggregated from headers (CONCEPT.md D37): their
- * releases, the files with their segment lists, and one completed binary per
- * file, as worker:file would have left it, so par2 naming, inspection and GC
- * treat them like any other file. A batch is a few statements, not one per
- * article.
+ * Stores what seeding nodes found (CONCEPT.md D37): per complete file its
+ * release, the file with its segment list, and one completed binary, as
+ * worker:file would have left it, so par2 naming, inspection and GC treat
+ * it like any other file; fragments take the regular ingest path. A batch
+ * is a few statements, not one per article.
  */
 class SeedService {
-  async store(files: AggregatedFile<IngestSegment>[]): Promise<SeedStoreResult> {
+  async store(batch: SeedFile[]): Promise<SeedStoreResult> {
     const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0 }
+    // A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
+    const files = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
     if (files.length === 0) return result
     const knex = db.connection().getWriteClient()
     const now = new Date()
+    const postedOf = (file: SeedFile) => (file.postedAt ? new Date(file.postedAt) : undefined)
 
-    // A file's identity and release come from its first segment, its date from the earliest.
-    const firstOf = (file: AggregatedFile<IngestSegment>) => file.segments[0]
-    const postedOf = (file: AggregatedFile<IngestSegment>) => {
-      const dates = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
-      return dates.length > 0 ? new Date(Math.min(...dates)) : undefined
-    }
     const releaseIds = await IngestService.upsertReleases(
       knex,
-      files.map((file) => ({ ...firstOf(file), postedAt: postedOf(file) })),
+      files.map((file) => ({ release: file.release, poster: file.poster, groups: file.groups, postedAt: postedOf(file) })),
       now
     )
     result.releases = releaseIds.size
 
-    const rows = files.map((file) => {
-      const first = firstOf(file)
-      const postedAt = postedOf(file)
-      return {
-        file_key: file.fileKey,
-        filename: first.name,
-        poster: first.poster,
-        date: (postedAt ?? now).getTime(),
-        parts: file.segments.length,
-        segments: file.segments.map((s) => ({ number: s.number, id: s.messageId, size: s.bytes })),
-        groups: JSON.stringify(first.groups),
-        release_id: first.release ? (releaseIds.get(first.release.key) ?? null) : null,
-        size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
-        created_at: now,
-        updated_at: now,
-      }
-    })
+    const rows = files.map((file) => ({
+      file_key: file.fileKey,
+      filename: file.name,
+      poster: file.poster,
+      date: (postedOf(file) ?? now).getTime(),
+      parts: file.segments.length,
+      segments: file.segments.map(([number, id, size]) => ({ number, id, size })),
+      groups: JSON.stringify(file.groups),
+      release_id: file.release ? (releaseIds.get(file.release.key) ?? null) : null,
+      size: file.segments.reduce((sum, [, , size]) => sum + size, 0),
+      created_at: now,
+      updated_at: now,
+    }))
     const inserted = await FileStore.insert(rows)
     result.files = inserted.length
     result.existing = rows.length - inserted.length
@@ -67,19 +61,19 @@ class SeedService {
       fileIds.set(row.file_key, Number(row.id))
     }
     // The completed binary worker:file would have left behind; an existing one just learns its file.
-    const binaries = rows.map((row, i) => ({
-      file_key: row.file_key,
-      poster: row.poster,
-      subject: firstOf(files[i]).subject,
-      name: row.filename,
-      total_parts: firstOf(files[i]).totalParts,
-      groups: row.groups,
-      posted_at: postedOf(files[i]) ?? null,
+    const binaries = files.map((file, i) => ({
+      file_key: file.fileKey,
+      poster: file.poster,
+      subject: file.subject,
+      name: file.name,
+      total_parts: file.totalParts,
+      groups: rows[i].groups,
+      posted_at: postedOf(file) ?? null,
       first_seen_at: now,
       last_seen_at: now,
       completed_at: now,
-      release_id: row.release_id,
-      file_id: fileIds.get(row.file_key) ?? null,
+      release_id: rows[i].release_id,
+      file_id: fileIds.get(file.fileKey) ?? null,
     }))
     binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
     await knex('binaries')
@@ -101,6 +95,16 @@ class SeedService {
     result.inspections = candidates.length
     return result
   }
+
+  /** Segments of files that didn't complete on a node: staging (D35) for lone ones, binaries and parts for the rest. */
+  async storeFragments(group: string, fragments: SeedFragment[]): Promise<void> {
+    if (fragments.length === 0) return
+    const segments = fragments.map(fromFragment)
+    const { ingest } = StagingService.enabled(group)
+      ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(group) })
+      : { ingest: segments }
+    await IngestService.ingest(ingest)
+  }
 }
 
 export default new SeedService()