Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker

- groups:add / ranges:plan / nodes:create / coordinator:run commands
- the node API: lease, the range's chunks as one framed stream from the
  spool, batches, heartbeat, finish, release (node guard, token per node)
- edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for
  the rest), posts a batch every 10 chunks, gives its range back on SIGTERM
- releases are gathered per batch: 1.04 chunk rows per release at 10
  chunks per batch (1.35 when built per XOVER chunk)
- partitions are checked per batch instead of remembered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/prefetcher.ts

@@ -0,0 +1,90 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+import NntpPool from '#services/nntp_pool'
+import Spool from '#services/spool'
+
+interface Target {
+  id: number
+  group: string
+  start: number
+  end: number
+  next: number
+  leased: boolean
+}
+
+/**
+ * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch):
+ * the missing chunks of leased ranges first, then the next planned ranges,
+ * about SPOOL_AHEAD_PER_LEASE per active lease (at least SPOOL_MIN_AHEAD),
+ * within SPOOL_MAX_GB. One range per connection at a time, in order.
+ */
+export class Prefetcher {
+  private busy = new Set<number>()
+  private stopped = false
+
+  /** One round: starts work for free connections; returns how many ranges it started. */
+  async tick(): Promise<number> {
+    const free = NntpPool.size - this.busy.size
+    if (free <= 0) return 0
+    const targets = await this.targets(free)
+    for (const target of targets) {
+      this.busy.add(target.id)
+      this.spoolRange(target)
+        .catch((error) => logger.warn(`Spooling range ${target.id}: ${error.message}`))
+        .finally(() => this.busy.delete(target.id))
+    }
+    return targets.length
+  }
+
+  stop() {
+    this.stopped = true
+  }
+
+  get active() {
+    return this.busy.size
+  }
+
+  private async targets(limit: number): Promise<Target[]> {
+    const busy = [...this.busy]
+    const select = (q: any) =>
+      q
+        .from('ranges as r')
+        .join('groups as g', 'g.id', 'r.group_id')
+        .whereNull('r.spooled_at')
+        .whereNotIn('r.id', busy.length > 0 ? busy : [0])
+        .select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.status')
+    // Leased ranges whose chunks aren't all there: a worker is waiting for them.
+    const leased = await select(db.query()).where('r.status', 'leased').orderBy('r.leased_at').limit(limit)
+    const out: Target[] = leased.map(toTarget)
+    if (out.length >= limit) return out
+
+    const [{ leases }] = await db.from('ranges').where('status', 'leased').count('* as leases')
+    const [{ ahead }] = await db.from('ranges').where('status', 'planned').where('spooled_chunks', '>', 0).count('* as ahead')
+    const wanted = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * Number(leases)) - Number(ahead)
+    if (wanted <= 0 || (await Spool.size()) >= coordinatorConfig.spool.maxBytes) return out
+    const planned = await select(db.query()).where('r.status', 'planned').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(Math.min(limit - out.length, wanted))
+    return [...out, ...planned.map(toTarget)]
+  }
+
+  private async spoolRange(target: Target) {
+    let count = 0
+    for (const span of Spool.chunksOf({ start: target.start, end: target.end }, target.next)) {
+      if (this.stopped) return
+      if (!(await Spool.has(target.id, span))) {
+        const data = await NntpPool.use(async (client) => {
+          await client.group(target.group)
+          return client.xoverRaw(span.from, span.to)
+        })
+        await Spool.write(target.id, span, data)
+      }
+      count++
+      await db.from('ranges').where('id', target.id).update({ spooled_chunks: count })
+    }
+    await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+  }
+}
+
+function toTarget(row: any): Target {
+  return { id: Number(row.id), group: row.group, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id), leased: row.status === 'leased' }
+}