Seed orchestrator: remote nodes lease ranges and post results over HTTPS

- seed_nodes (hashed bearer tokens) and seed_ranges (planned ranges with
  lease, resume point and counts); migration …140.
- SeedOrchestrator: lease (oldest plan first, FOR UPDATE SKIP LOCKED; a
  lease without a batch for 15 min passes to the next node and resumes at
  next_id), batch (stores files and fragments, moves next_id), finish,
  release, plan (from below live indexing / the backfill / earlier plans
  down to N days back; takes over the group's backfill), status.
- Seed API under /api/v1/seed (lease, ranges/:id/batches|done|release,
  status) behind a node-token middleware; JSON bodies up to 64 MB after
  gzip.
- seed:node runs a node: no database or Redis, outgoing HTTPS only, N
  connections, gzip batches, retries for network errors and 5xx, drops a
  range on 409, gives its ranges back on SIGTERM. seed-node/ has a compose
  file that builds the image from this repository.
- seed:token, seed:plan, seed:status for the operator. Header parsing moved
  to lib/ingest and the range loop to lib/seed/runner, shared by seed:range
  and seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/ingest/segment.ts

@@ -0,0 +1,105 @@
+import { parseXrefGroups } from '../nntp/xref.js'
+import { fileKey, parseSubject, weakFileKey } from '../subject/index.js'
+import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
+import { inferTotalParts, type YencMeta } from '../yenc/index.js'
+
+/** One segment, with the identity of the file it belongs to already resolved. */
+export interface IngestSegment {
+  fileKey: string
+  poster: string
+  /** Subject without segment counter. */
+  subject: string
+  name: string
+  totalParts: number
+  number: number
+  /** Without angle brackets. */
+  messageId: string
+  bytes: number
+  articleNumber?: bigint | number
+  postedAt?: Date
+  groups: string[]
+  /**
+   * The identity comes from the yEnc name only (obfuscated posts). Such names
+   * can be shared by several equal-sized files of one post, so a second
+   * article for the same part number marks the binary as conflicted.
+   */
+  weakIdentity?: boolean
+  /** Release (download) this file belongs to, when it can be determined. */
+  release?: ReleaseIdentity | null
+  /** Exact file size from =ybegin (body-derived segments only). */
+  size?: number
+  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
+  pcrc32?: string
+  partSize?: number
+  fileCrc32?: string
+  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
+  md5_16k?: string
+}
+
+/** An XOVER overview entry as returned by nntp-js. */
+export type OverviewHeader = Record<string, string>
+
+function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
+  const xrefGroups = parseXrefGroups(header.xref)
+  return {
+    poster: header.from ?? '',
+    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
+    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
+    articleNumber,
+    postedAt: header.date ? new Date(header.date) : undefined,
+    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
+  }
+}
+
+/** Builds a segment from the subject line; null when the subject is obfuscated. */
+export function segmentFromHeader(
+  header: OverviewHeader,
+  groupName: string,
+  articleNumber?: bigint | number
+): IngestSegment | null {
+  const parsed = header.subject ? parseSubject(header.subject) : null
+  if (!parsed || !header['message-id']) return null
+  const base = common(header, groupName, articleNumber)
+  return {
+    ...base,
+    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
+    subject: parsed.base,
+    name: parsed.name,
+    totalParts: parsed.totalSegments,
+    number: parsed.segment,
+    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
+  }
+}
+
+/**
+ * Builds a segment for an obfuscated post from the yEnc header in its body.
+ * Subject and poster can be random per article, so the identity comes from
+ * the yEnc header only (see weakFileKey).
+ */
+export function segmentFromYenc(
+  header: OverviewHeader,
+  groupName: string,
+  meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
+  articleNumber?: bigint | number
+): IngestSegment | null {
+  const base = common(header, groupName, articleNumber)
+  const totalParts = inferTotalParts(meta)
+  const number = meta.begin.part ?? 1
+  if (!base.messageId || number < 1 || number > totalParts) return null
+  const subject = `"${meta.begin.name}" yEnc`
+  return {
+    ...base,
+    fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
+    subject,
+    name: meta.begin.name,
+    totalParts,
+    number,
+    weakIdentity: true,
+    release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
+    size: meta.begin.size,
+    // Single-part posts only carry crc32, which is then also the part's CRC.
+    pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
+    partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
+    fileCrc32: meta.end?.crc32,
+  }
+}

lib/seed/runner.ts

@@ -0,0 +1,78 @@
+import type { XoverClient, Overview } from '../nntp/xover_client.js'
+import { segmentFromHeader, type IngestSegment, type OverviewHeader } from '../ingest/segment.js'
+import { FileAggregator } from '../aggregate/index.js'
+import { toFragment, toSeedFile, type SeedBatch } from './wire.js'
+
+export interface RunRangeOptions {
+  client: XoverClient
+  group: string
+  /** Inclusive article numbers. */
+  start: number
+  end: number
+  /** Articles per XOVER. */
+  chunk?: number
+  /** Batches without a new part before an open file leaves as fragments. */
+  idle?: number
+  /** XOVER batches per SeedBatch handed to `sink`. */
+  submitEvery?: number
+  /** Headers left to the deferred-body path (counted, not indexed). */
+  isDeferred?: (header: OverviewHeader) => boolean
+  /** Receives batches in order; the next XOVER is already in flight meanwhile. */
+  sink: (batch: SeedBatch) => Promise<void>
+}
+
+/**
+ * Indexes one article range on a seeding node (CONCEPT.md D37): XOVER in
+ * chunks over one connection (the next chunk in flight while the previous
+ * one is handled), headers parsed with the live rules, grouped into files
+ * in memory; complete files and given-up fragments go to `sink` in order.
+ * Shared by seed:range (stores directly) and seed:node (submits over HTTPS).
+ */
+export async function runRange(options: RunRangeOptions): Promise<void> {
+  const chunk = options.chunk ?? 20_000
+  const idle = options.idle ?? 5
+  const submitEvery = options.submitEvery ?? 5
+  const aggregator = new FileAggregator<IngestSegment>()
+  await options.client.group(options.group)
+
+  const fetch = (from: number) => {
+    const promise = options.client.xover(from, Math.min(from + chunk - 1, options.end))
+    // Awaited after the current batch is handled; until then a failure must not count as unhandled.
+    promise.catch(() => {})
+    return promise
+  }
+
+  let batch = emptyBatch(options.start)
+  let pending = 0
+  let from = options.start
+  let next: Promise<Overview[]> | null = fetch(from)
+  while (next) {
+    const overviews = await next
+    from += chunk
+    next = from <= options.end ? fetch(from) : null
+
+    const segments: IngestSegment[] = []
+    for (const [articleNumber, header] of overviews) {
+      if (!Number.isFinite(articleNumber) || !header['message-id']) continue
+      const segment = segmentFromHeader(header, options.group, articleNumber)
+      if (segment) segments.push(segment)
+      else if (options.isDeferred?.(header)) batch.deferred++
+      else batch.obfuscated++
+    }
+    batch.headers += overviews.length
+    batch.files.push(...aggregator.add(segments).map(toSeedFile))
+    batch.fragments.push(...aggregator.evict(idle).map(toFragment))
+    batch.next = Math.min(from, options.end + 1)
+
+    if (++pending >= submitEvery || !next) {
+      if (!next) batch.fragments.push(...aggregator.evict(0).map(toFragment))
+      await options.sink(batch)
+      batch = emptyBatch(batch.next)
+      pending = 0
+    }
+  }
+}
+
+function emptyBatch(next: number): SeedBatch {
+  return { next, headers: 0, obfuscated: 0, deferred: 0, files: [], fragments: [] }
+}

lib/seed/wire.ts

@@ -0,0 +1,73 @@
+import type { IngestSegment } from '../ingest/segment.js'
+import type { ReleaseIdentity } from '../release/index.js'
+import type { AggregatedFile } from '../aggregate/index.js'
+
+/**
+ * A complete file as a seeding node reports it (CONCEPT.md D37): what the
+ * file, its release and its binary need, once per file, and the segments
+ * as [part number, message-id, bytes].
+ */
+export interface SeedFile {
+  fileKey: string
+  name: string
+  poster: string
+  subject: string
+  totalParts: number
+  groups: string[]
+  /** Earliest post date of its articles, ISO 8601. */
+  postedAt: string | null
+  release: ReleaseIdentity | null
+  segments: [number, string, number][]
+}
+
+/** A segment of a file that didn't complete on the node, for the regular ingest path. */
+export type SeedFragment = Omit<IngestSegment, 'postedAt' | 'articleNumber'> & { postedAt?: string; articleNumber?: number }
+
+/** What a node sends per batch: the article it has reached, and what it found up to there. */
+export interface SeedBatch {
+  /** First article number not covered yet (everything below it, from the range start, is in this or earlier batches). */
+  next: number
+  headers: number
+  obfuscated: number
+  deferred: number
+  files: SeedFile[]
+  fragments: SeedFragment[]
+}
+
+export function toSeedFile(file: AggregatedFile<IngestSegment>): SeedFile {
+  const first = file.segments[0]
+  const times = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
+  return {
+    fileKey: file.fileKey,
+    name: first.name,
+    poster: first.poster,
+    subject: first.subject,
+    totalParts: first.totalParts,
+    groups: first.groups,
+    postedAt: times.length > 0 ? new Date(Math.min(...times)).toISOString() : null,
+    release: first.release ?? null,
+    segments: file.segments.map((s) => [s.number, s.messageId, s.bytes]),
+  }
+}
+
+export function toFragment(segment: IngestSegment): SeedFragment {
+  const { postedAt, articleNumber, ...rest } = segment
+  const date = postedAt && !Number.isNaN(postedAt.getTime()) ? postedAt.toISOString() : undefined
+  return { ...rest, postedAt: date, articleNumber: articleNumber === undefined ? undefined : Number(articleNumber) }
+}
+
+export function fromFragment(fragment: SeedFragment): IngestSegment {
+  // The JSON body parser turns empty strings into null; the columns want ''.
+  return {
+    ...fragment,
+    poster: fragment.poster ?? '',
+    subject: fragment.subject ?? '',
+    name: fragment.name ?? '',
+    postedAt: fragment.postedAt ? new Date(fragment.postedAt) : undefined,
+  }
+}
+
+/** A file from a node's JSON, with empty strings back where the body parser made them null. */
+export function normalizeSeedFile(file: SeedFile): SeedFile {
+  return { ...file, name: file.name ?? '', poster: file.poster ?? '', subject: file.subject ?? '' }
+}