Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker

- groups:add / ranges:plan / nodes:create / coordinator:run commands
- the node API: lease, the range's chunks as one framed stream from the
  spool, batches, heartbeat, finish, release (node guard, token per node)
- edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for
  the rest), posts a batch every 10 chunks, gives its range back on SIGTERM
- releases are gathered per batch: 1.04 chunk rows per release at 10
  chunks per batch (1.35 when built per XOVER chunk)
- partitions are checked per batch instead of remembered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/coordinator_run.ts

@@ -0,0 +1,51 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { Prefetcher } from '#services/prefetcher'
+import Planner from '#services/planner'
+import NntpPool from '#services/nntp_pool'
+
+const TICK_MS = 2000
+const PLAN_EVERY_MS = 60 * 60 * 1000
+
+/**
+ * The coordinator's background work (REBUILD.md, part 1): the prefetcher,
+ * and planning ranges for new articles every hour. The API (leases, stream,
+ * batches) runs in the web server.
+ */
+export default class CoordinatorRun extends BaseCommand {
+  static commandName = 'coordinator:run'
+  static description = 'Prefetches ranges into the spool and plans new ones'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  async run() {
+    const prefetcher = new Prefetcher()
+    let stopping = false
+    let stopped: () => void
+    const done = new Promise<void>((resolve) => (stopped = resolve))
+    // bin/console turns SIGTERM into app.terminate: wait for the spooling in flight first.
+    this.app.terminating(async () => {
+      stopping = true
+      prefetcher.stop()
+      await done
+    })
+    let plannedAt = 0
+    this.logger.info('Coordinator started.')
+    while (!stopping) {
+      try {
+        if (Date.now() - plannedAt >= PLAN_EVERY_MS) {
+          plannedAt = Date.now()
+          const { groups, ranges } = await Planner.planAll()
+          if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
+        }
+        await prefetcher.tick()
+      } catch (error: any) {
+        this.logger.error(`Coordinator round failed: ${error.message}`)
+      }
+      await sleep(TICK_MS)
+    }
+    while (prefetcher.active > 0) await sleep(200)
+    await NntpPool.close()
+    stopped!()
+  }
+}

commands/edge_bench.ts

@@ -37,6 +37,9 @@ export default class EdgeBench extends BaseCommand {
   @flags.number({ description: 'Articles per XOVER', default: 50_000 })
   declare chunk: number
 
+  @flags.number({ description: 'Chunks per batch (results taken every N chunks)', default: 10 })
+  declare batchChunks: number
+
   @flags.number({ description: 'Chunks without a new part before an open file is given up', default: 5 })
   declare idle: number
 
@@ -87,6 +90,8 @@ export default class EdgeBench extends BaseCommand {
     let peakRss = 0
     const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
     const counts = new Map<string, number>()
+    const keys = new Set<string>()
+    let batches = 0
     const cpuStart = process.cpuUsage()
     const wall = Date.now()
     for (const file of files) {
@@ -102,12 +107,18 @@ export default class EdgeBench extends BaseCommand {
       at = performance.now()
       const next = Number(file.split('-')[2].replace('.gz', '')) + 1
       indexer.add(overviews, next)
-      if (file === files.at(-1)) indexer.finish()
+      const last = file === files.at(-1)
+      if (last) indexer.finish()
+      t.index += performance.now() - at
+      if (!last && (files.indexOf(file) + 1) % this.batchChunks !== 0) continue
+      at = performance.now()
       const batch = indexer.take()
       t.index += performance.now() - at
       out.headers += batch.headers
       out.releases += batch.releases.length
+      batches++
       for (const r of batch.releases) {
+        keys.add(r.key)
         out.files += r.files
         out.segments += r.segments
         out.payload += r.payload.length
@@ -124,6 +135,7 @@ export default class EdgeBench extends BaseCommand {
     this.logger.info(`${group} ${start}–${end}: ${files.length} chunks, ${(compressed / 1e6).toFixed(0)} MB compressed, ${(text / 1e6).toFixed(0)} MB text`)
     this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
     this.logger.info(`  inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB`)
+    this.logger.info(`  ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
     this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
     for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(`  dropped ${k}: ${v.toLocaleString('en')} articles`)
   }

commands/edge_node.ts

@@ -0,0 +1,157 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { unzipSync } from 'node:zlib'
+import { parseOverviews } from '#lib/nntp/index'
+import { RangeIndexer, decodeFrames, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
+
+const IDLE_MS = 30_000
+const RETRIES = 8
+
+class LeaseLost extends Error {}
+
+/**
+ * An edge worker (REBUILD.md, part 2): leases a range from the coordinator,
+ * streams its chunks, indexes them, posts a batch every --batch-chunks
+ * chunks (each moves the range's resume point), and finishes the range.
+ * Needs COORDINATOR_URL and NODE_TOKEN only. On SIGTERM it gives its range
+ * back after the current chunk; the next worker resumes from the last batch.
+ */
+export default class EdgeNode extends BaseCommand {
+  static commandName = 'edge:node'
+  static description = 'Indexes ranges for the coordinator'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.number({ description: 'Chunks per posted batch (500k articles: ~4% of releases get a second chunk; LOG.md 2026-10-09)', default: 10 })
+  declare batchChunks: number
+
+  @flags.number({ description: 'Stop after this many ranges (default: keep going)' })
+  declare ranges?: number
+
+  private stopping = false
+
+  async run() {
+    const url = process.env.COORDINATOR_URL
+    const token = process.env.NODE_TOKEN
+    if (!url || !token) throw new Error('COORDINATOR_URL and NODE_TOKEN are needed')
+    let stopped: () => void
+    const done = new Promise<void>((resolve) => (stopped = resolve))
+    this.app.terminating(async () => {
+      this.stopping = true
+      await done
+    })
+    const api = new Api(url.replace(/\/$/, ''), token)
+    let ranges = 0
+    try {
+      while (!this.stopping && (!this.ranges || ranges < this.ranges)) {
+        let lease: Lease | null = null
+        try {
+          lease = await api.lease()
+        } catch (error: any) {
+          this.logger.warning(`Lease failed: ${error.message}`)
+        }
+        if (!lease) {
+          if (this.ranges) break
+          await sleep(IDLE_MS)
+          continue
+        }
+        await this.runRange(api, lease)
+        ranges++
+      }
+    } finally {
+      stopped!()
+    }
+    await this.terminate()
+  }
+
+  private async runRange(api: Api, lease: Lease) {
+    const { range, rules } = lease
+    const started = Date.now()
+    const indexer = new RangeIndexer({ group: range.group, start: range.start, end: range.end, rules })
+    const totals = { headers: 0, releases: 0, candidates: 0 }
+    let chunks = 0
+    const post = async (batch: EdgeBatch) => {
+      const stored = await api.batch(range.id, batch)
+      totals.headers += batch.headers
+      totals.releases += stored.releases
+      totals.candidates += stored.candidates
+    }
+    try {
+      for await (const frame of decodeFrames(await api.stream(range.id))) {
+        indexer.add(parseOverviews(unzipSync(frame.data).toString('utf8')), frame.to + 1)
+        chunks++
+        const last = frame.to >= range.end
+        if (last) indexer.finish()
+        if (last || chunks % this.batchChunks === 0) await post(indexer.take())
+        if (this.stopping && !last) {
+          // Back to the queue; the next worker resumes at the last batch.
+          if (chunks % this.batchChunks !== 0) this.logger.info(`Range ${range.id}: giving it back (${chunks % this.batchChunks} chunks since the last batch are read again).`)
+          await api.call(range.id, 'release')
+          return
+        }
+      }
+      await api.call(range.id, 'finish')
+      const seconds = (Date.now() - started) / 1000
+      this.logger.info(
+        `${range.group} ${range.start}–${range.end}: ${totals.headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s ` +
+          `(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} release chunks, ${totals.candidates} probe candidates.`
+      )
+    } catch (error: any) {
+      if (error instanceof LeaseLost) {
+        this.logger.warning(`Range ${range.id}: ${error.message}`)
+        return
+      }
+      this.logger.error(`Range ${range.id} failed: ${error.message}; giving it back.`)
+      await api.call(range.id, 'release').catch(() => {})
+    }
+  }
+}
+
+/** The coordinator's node API, with retries for network trouble and 5xx. */
+class Api {
+  constructor(
+    private base: string,
+    private token: string
+  ) {}
+
+  private async request(path: string, init: RequestInit = {}): Promise<Response> {
+    for (let attempt = 1; ; attempt++) {
+      try {
+        const response = await fetch(`${this.base}/api/v1/node/${path}`, {
+          ...init,
+          headers: { 'Authorization': `Bearer ${this.token}`, 'Content-Type': 'application/json', 'Accept': 'application/json', ...init.headers },
+        })
+        if (response.status === 409) throw new LeaseLost(((await response.json().catch(() => ({}))) as { error?: string }).error ?? 'lease lost')
+        if (response.status < 500) return response
+        if (attempt >= RETRIES) throw new Error(`${path}: HTTP ${response.status}`)
+      } catch (error) {
+        if (error instanceof LeaseLost || attempt >= RETRIES) throw error
+      }
+      await sleep(Math.min(60_000, 1000 * 2 ** attempt))
+    }
+  }
+
+  async lease(): Promise<Lease | null> {
+    const response = await this.request('lease', { method: 'POST', body: '{}' })
+    if (response.status === 204) return null
+    if (!response.ok) throw new Error(`lease: HTTP ${response.status}`)
+    return response.json() as Promise<Lease>
+  }
+
+  async stream(rangeId: number): Promise<AsyncIterable<Uint8Array>> {
+    const response = await this.request(`ranges/${rangeId}/stream`, { headers: { Accept: 'application/octet-stream' } })
+    if (!response.ok || !response.body) throw new Error(`stream: HTTP ${response.status}`)
+    return response.body as unknown as AsyncIterable<Uint8Array>
+  }
+
+  async batch(rangeId: number, batch: EdgeBatch): Promise<{ releases: number; candidates: number }> {
+    const response = await this.request(`ranges/${rangeId}/batches`, { method: 'POST', body: JSON.stringify(toWire(batch)) })
+    if (!response.ok) throw new Error(`batch: HTTP ${response.status}`)
+    return response.json() as Promise<{ releases: number; candidates: number }>
+  }
+
+  async call(rangeId: number, action: 'finish' | 'release' | 'heartbeat') {
+    const response = await this.request(`ranges/${rangeId}/${action}`, { method: 'POST', body: '{}' })
+    if (!response.ok) throw new Error(`${action}: HTTP ${response.status}`)
+  }
+}

commands/groups_add.ts

@@ -0,0 +1,30 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Planner from '#services/planner'
+import NntpPool from '#services/nntp_pool'
+
+/** Adds groups to index, after the existing ones in priority (owner rule). */
+export default class GroupsAdd extends BaseCommand {
+  static commandName = 'groups:add'
+  static description = 'Adds groups to index (by name, or the largest by LIST ACTIVE)'
+  static options: CommandOptions = { startApp: true }
+
+  @args.spread({ description: 'Group names', required: false })
+  declare names?: string[]
+
+  @flags.number({ description: 'The largest N groups matching --pattern' })
+  declare top?: number
+
+  @flags.string({ description: 'LIST ACTIVE pattern for --top', default: 'alt.binaries.*' })
+  declare pattern: string
+
+  async run() {
+    try {
+      const names = this.top ? await Planner.largestGroups(this.pattern, this.top) : (this.names ?? [])
+      const added = await Planner.addGroups(names)
+      this.logger.info(`${added} of ${names.length} groups added.`)
+    } finally {
+      await NntpPool.close()
+    }
+  }
+}

commands/nodes_create.ts

@@ -0,0 +1,19 @@
+import { BaseCommand, args } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Node from '#models/node'
+
+/** Registers an edge worker and prints its token (shown once). */
+export default class NodesCreate extends BaseCommand {
+  static commandName = 'nodes:create'
+  static description = 'Registers an edge worker and prints its token'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'Node name' })
+  declare name: string
+
+  async run() {
+    const node = await Node.firstOrCreate({ name: this.name }, { name: this.name })
+    const token = await Node.accessTokens.create(node)
+    this.logger.info(`Node ${node.name} (#${node.id}). Token: ${token.value!.release()}`)
+  }
+}

commands/ranges_plan.ts

@@ -0,0 +1,20 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Planner from '#services/planner'
+import NntpPool from '#services/nntp_pool'
+
+/** Cuts every active group's history (and new articles since) into ranges. */
+export default class RangesPlan extends BaseCommand {
+  static commandName = 'ranges:plan'
+  static description = "Plans ranges for the indexed groups' articles"
+  static options: CommandOptions = { startApp: true }
+
+  async run() {
+    try {
+      const { groups, ranges } = await Planner.planAll()
+      this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
+    } finally {
+      await NntpPool.close()
+    }
+  }
+}