Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end

A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail
planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades
through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads`
(EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The
lease order within a lane alternates the group with the most planned articles and the longest unheld one. The
takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range
is held by a holder gone quiet can be taken over.

`coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with
their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the
corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot
file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/controllers/node_controller.ts

@@ -3,18 +3,13 @@ import { Readable } from 'node:stream'
 import { setTimeout as sleep } from 'node:timers/promises'
 import db from '@adonisjs/lucid/services/db'
 import Leases, { LeaseLost } from '#services/leases'
-import GroupLeases from '#services/group_leases'
+import GroupLeases, { LANES } from '#services/group_leases'
 import BatchStore, { StaleBatch } from '#services/batch_store'
 import Spool, { type ChunkSpan } from '#services/spool'
 import Corpus from '#services/corpus'
-import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
+import coordinatorConfig from '#config/coordinator'
+import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
 
-/**
- * How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
- * coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
- * (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
- */
-const CHUNK_WAIT_MS = 60 * 60 * 1000
 /** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
 const KEEPALIVE_MS = 20_000
 
@@ -35,16 +30,29 @@ export default class NodeController {
 
   // Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
   // ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
+  // `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
+
+  /** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
+  private lane(ctx: HttpContext): Lane | null {
+    const lane = ctx.request.qs().lane ?? 'history'
+    if (LANES.includes(lane)) return lane
+    ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
+    return null
+  }
 
   async leaseGroup(ctx: HttpContext) {
-    const lease = await GroupLeases.lease(this.node(ctx))
+    const lane = this.lane(ctx)
+    if (!lane) return
+    const lease = await GroupLeases.lease(this.node(ctx), lane)
     if (!lease) return ctx.response.noContent()
     return lease
   }
 
   async nextRange(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
     try {
-      const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx))
+      const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
       if (!next) return ctx.response.noContent()
       return next
     } catch (error) {
@@ -54,11 +62,15 @@ export default class NodeController {
   }
 
   async groupHeartbeat(ctx: HttpContext) {
-    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId))
+    const lane = this.lane(ctx)
+    if (!lane) return
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
   }
 
   async releaseGroup(ctx: HttpContext) {
-    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId))
+    const lane = this.lane(ctx)
+    if (!lane) return
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
   }
 
   /**
@@ -66,16 +78,20 @@ export default class NodeController {
    * (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
    */
   async putSnapshot(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
     const next = Number(ctx.request.qs().next)
     if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
     const raw = ctx.request.raw()
     const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
-    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body))
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
   }
 
   async getSnapshot(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
     try {
-      const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx))
+      const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
       if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
       ctx.response.header('Content-Type', 'application/octet-stream')
       ctx.response.header('Content-Length', String(snapshot.size))
@@ -107,7 +123,7 @@ export default class NodeController {
       corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
     async function* frames() {
       for (const span of spans) {
-        const waitUntil = Date.now() + CHUNK_WAIT_MS
+        const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
         let quietSince = Date.now()
         let data = await stored(span)
         while (!data) {
@@ -152,7 +168,7 @@ export default class NodeController {
     return this.lifecycle(ctx, async (rangeId, nodeId) => {
       const range = await Leases.leased(rangeId, nodeId)
       await Leases.finish(rangeId, nodeId)
-      await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers))
+      await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
     })
   }