Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker

- groups:add / ranges:plan / nodes:create / coordinator:run commands
- the node API: lease, the range's chunks as one framed stream from the
  spool, batches, heartbeat, finish, release (node guard, token per node)
- edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for
  the rest), posts a batch every 10 chunks, gives its range back on SIGTERM
- releases are gathered per batch: 1.04 chunk rows per release at 10
  chunks per batch (1.35 when built per XOVER chunk)
- partitions are checked per batch instead of remembered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/edge/index.ts

@@ -1 +1,4 @@
 export { RangeIndexer, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
+export { encodeFrame, decodeFrames, type Frame } from './stream.js'
+export { toWire, fromWire, type WireBatch } from './wire.js'
+export type { Lease } from './lease.js'

lib/edge/lease.ts

@@ -0,0 +1,7 @@
+import type { Rules } from '../rules/index.js'
+
+/** What a worker gets with a range (the coordinator's lease). */
+export interface Lease {
+  range: { id: number; group: string; start: number; end: number; next: number }
+  rules: Rules
+}

lib/edge/range_indexer.ts

@@ -143,6 +143,7 @@ export class RangeIndexer {
 
   /** What was found since the last take, and a fresh batch. */
   take(): EdgeBatch {
+    this.judgeReleases()
     const out = this.batch
     out.counts = [...this.counts.values()]
     this.counts.clear()
@@ -154,14 +155,22 @@ export class RangeIndexer {
     return this.aggregator.openFiles
   }
 
+  /** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
+  private completed = new Map<string, AggregatedFile<IngestSegment>[]>()
+
   private files(files: AggregatedFile<IngestSegment>[]) {
-    const byRelease = new Map<string, AggregatedFile<IngestSegment>[]>()
     for (const file of files) {
       const key = releaseKeyOf(file.segments[0])
-      const list = byRelease.get(key)
+      const list = this.completed.get(key)
       if (list) list.push(file)
-      else byRelease.set(key, [file])
+      else this.completed.set(key, [file])
     }
+  }
+
+  /** Judges and encodes the releases completed in this batch. */
+  private judgeReleases() {
+    const byRelease = this.completed
+    this.completed = new Map()
     for (const [key, list] of byRelease) {
       const verdict = this.rules.judge(
         key,

lib/edge/stream.ts

@@ -0,0 +1,35 @@
+/**
+ * A range's chunks on the wire (REBUILD.md, prefetch): one streaming
+ * response, a frame per chunk: from (8 bytes), to (8 bytes), length
+ * (4 bytes), then the chunk as the news server sent it (compressed XOVER).
+ */
+export interface Frame {
+  from: number
+  to: number
+  data: Buffer
+}
+
+const HEADER = 20
+
+export function encodeFrame(frame: Frame): Buffer {
+  const head = Buffer.alloc(HEADER)
+  head.writeBigUInt64BE(BigInt(frame.from), 0)
+  head.writeBigUInt64BE(BigInt(frame.to), 8)
+  head.writeUInt32BE(frame.data.length, 16)
+  return Buffer.concat([head, frame.data])
+}
+
+/** The frames of a byte stream, as they complete. */
+export async function* decodeFrames(stream: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): AsyncGenerator<Frame> {
+  let buffer = Buffer.alloc(0)
+  for await (const piece of stream) {
+    buffer = buffer.length === 0 ? Buffer.from(piece) : Buffer.concat([buffer, piece])
+    while (buffer.length >= HEADER) {
+      const length = buffer.readUInt32BE(16)
+      if (buffer.length < HEADER + length) break
+      yield { from: Number(buffer.readBigUInt64BE(0)), to: Number(buffer.readBigUInt64BE(8)), data: buffer.subarray(HEADER, HEADER + length) }
+      buffer = buffer.subarray(HEADER + length)
+    }
+  }
+  if (buffer.length > 0) throw new Error(`Stream ended inside a frame (${buffer.length} bytes left)`)
+}

lib/edge/wire.ts

@@ -0,0 +1,23 @@
+import type { EdgeBatch } from './range_indexer.js'
+
+/** An EdgeBatch as JSON: chunk payloads in base64. */
+export type WireBatch = Omit<EdgeBatch, 'releases' | 'samples'> & {
+  releases: (Omit<EdgeBatch['releases'][number], 'payload'> & { payload: string })[]
+  samples: (Omit<EdgeBatch['samples'][number], 'payload'> & { payload: string })[]
+}
+
+export function toWire(batch: EdgeBatch): WireBatch {
+  return {
+    ...batch,
+    releases: batch.releases.map((r) => ({ ...r, payload: r.payload.toString('base64') })),
+    samples: batch.samples.map((s) => ({ ...s, payload: s.payload.toString('base64') })),
+  }
+}
+
+export function fromWire(wire: WireBatch): EdgeBatch {
+  return {
+    ...wire,
+    releases: wire.releases.map((r) => ({ ...r, payload: Buffer.from(r.payload, 'base64') })),
+    samples: wire.samples.map((s) => ({ ...s, payload: Buffer.from(s.payload, 'base64') })),
+  }
+}

lib/nntp/xover_client.ts

@@ -98,6 +98,21 @@ export class XoverClient {
    * compresses (plain replies are deflated here): the seed API passes these to nodes
    * without unpacking them (CONCEPT.md D38). `parseOverviews(unzipSync(raw).toString())`.
    */
+  /** LIST ACTIVE <pattern>: each group's name, high and low article (low is often just 2 on big servers; GROUP gives the real one). */
+  async listActive(pattern: string): Promise<{ name: string; high: number; low: number }[]> {
+    return this.timed(async () => {
+      const reply = await this.command(`LIST ACTIVE ${pattern}`)
+      await this.expect(reply, ['215'])
+      return (await this.plainBody())
+        .split('\r\n')
+        .filter(Boolean)
+        .map((line) => {
+          const [name, high, low] = line.split(' ')
+          return { name, high: Number(high), low: Number(low) }
+        })
+    })
+  }
+
   async xoverRaw(from: number, to: number): Promise<Buffer> {
     return this.timed(async () => {
       const reply = await this.command(`XOVER ${from}-${to}`)