Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end

A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail
planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades
through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads`
(EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The
lease order within a lane alternates the group with the most planned articles and the longest unheld one. The
takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range
is held by a holder gone quiet can be taken over.

`coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with
their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the
corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot
file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/controllers/node_controller.ts

@@ -3,18 +3,13 @@ import { Readable } from 'node:stream'
 import { setTimeout as sleep } from 'node:timers/promises'
 import db from '@adonisjs/lucid/services/db'
 import Leases, { LeaseLost } from '#services/leases'
-import GroupLeases from '#services/group_leases'
+import GroupLeases, { LANES } from '#services/group_leases'
 import BatchStore, { StaleBatch } from '#services/batch_store'
 import Spool, { type ChunkSpan } from '#services/spool'
 import Corpus from '#services/corpus'
-import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
+import coordinatorConfig from '#config/coordinator'
+import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
 
-/**
- * How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
- * coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
- * (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
- */
-const CHUNK_WAIT_MS = 60 * 60 * 1000
 /** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
 const KEEPALIVE_MS = 20_000
 
@@ -35,16 +30,29 @@ export default class NodeController {
 
   // Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
   // ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
+  // `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
+
+  /** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
+  private lane(ctx: HttpContext): Lane | null {
+    const lane = ctx.request.qs().lane ?? 'history'
+    if (LANES.includes(lane)) return lane
+    ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
+    return null
+  }
 
   async leaseGroup(ctx: HttpContext) {
-    const lease = await GroupLeases.lease(this.node(ctx))
+    const lane = this.lane(ctx)
+    if (!lane) return
+    const lease = await GroupLeases.lease(this.node(ctx), lane)
     if (!lease) return ctx.response.noContent()
     return lease
   }
 
   async nextRange(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
     try {
-      const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx))
+      const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
       if (!next) return ctx.response.noContent()
       return next
     } catch (error) {
@@ -54,11 +62,15 @@ export default class NodeController {
   }
 
   async groupHeartbeat(ctx: HttpContext) {
-    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId))
+    const lane = this.lane(ctx)
+    if (!lane) return
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
   }
 
   async releaseGroup(ctx: HttpContext) {
-    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId))
+    const lane = this.lane(ctx)
+    if (!lane) return
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
   }
 
   /**
@@ -66,16 +78,20 @@ export default class NodeController {
    * (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
    */
   async putSnapshot(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
     const next = Number(ctx.request.qs().next)
     if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
     const raw = ctx.request.raw()
     const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
-    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body))
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
   }
 
   async getSnapshot(ctx: HttpContext) {
+    const lane = this.lane(ctx)
+    if (!lane) return
     try {
-      const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx))
+      const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
       if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
       ctx.response.header('Content-Type', 'application/octet-stream')
       ctx.response.header('Content-Length', String(snapshot.size))
@@ -107,7 +123,7 @@ export default class NodeController {
       corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
     async function* frames() {
       for (const span of spans) {
-        const waitUntil = Date.now() + CHUNK_WAIT_MS
+        const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
         let quietSince = Date.now()
         let data = await stored(span)
         while (!data) {
@@ -152,7 +168,7 @@ export default class NodeController {
     return this.lifecycle(ctx, async (rangeId, nodeId) => {
       const range = await Leases.leased(rangeId, nodeId)
       await Leases.finish(rangeId, nodeId)
-      await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers))
+      await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
     })
   }
 

app/services/group_leases.ts

@@ -8,120 +8,179 @@ import coordinatorConfig from '#config/coordinator'
 import RuleSets from '#services/rule_sets'
 import CatalogSpace from '#services/catalog_space'
 import { LeaseLost } from '#services/leases'
-import type { GroupLease, NextRange } from '#lib/edge/index'
+import type { GroupLease, Lane, NextRange } from '#lib/edge/index'
+
+export const LANES: Lane[] = ['history', 'tail']
 
 /**
  * Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
  * next range each time it finished one, so a group's ranges are indexed in article order through one carried
- * window. The coordinator keeps one row a group (group_leases): the holder, its heartbeat, the range it works and
- * the snapshot its window stands at.
+ * window. The coordinator keeps one row a group and lane (group_leases): the holder, its heartbeat, the range it
+ * works and the snapshot its window stands at.
  *
- * - lease: the free group with the most urgent planned range (the ranges' priority: the tail before history, then
- *   the windows), the longest unheld first; a holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group
- *   resets what the window's snapshot doesn't cover: the ranges from the snapshot's article on are planned again
- *   (the batch store's keys make the redo idempotent); without a snapshot the window is lost and the group
- *   continues at its resume point (one seam).
- * - next: the group's lowest planned range (or the one this node already has), leased to the node; none when the
- *   group is caught up: the worker snapshots and gives the group back, and gets it again when the tail planned
- *   more.
- * - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR), given to the next holder.
+ * - Two lanes (2026-10-27): the `history` lane works the ranges the bounds hand out (LEASE_MIN/MAX_PRIORITY,
+ *   LEASE_CAPTURE_ONLY: the windows, the history), the `tail` lane the tail planner's ranges (priority at or below
+ *   the tail's). Each lane has its own holder, window and snapshot, so a thread on the tail never wades through a
+ *   group's history first (a group's ranges come in article order, and a held group's history is days of ranges), and
+ *   the tail of a group whose history is held isn't stuck behind it. The one seam a group is where the history lane
+ *   reaches the tail lane's first range (its ranges are done: the history lane is caught up).
+ * - lease: the free group with the most urgent planned range of the lane (the ranges' priority), and among those a
+ *   mix: the group with the most planned articles and the longest unheld one (never held first) take turns, so a
+ *   big group snapshotted at a stop doesn't wait behind hundreds of small never-held ones, and a quiet group's tail
+ *   is visited all the same. A holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group resets what the
+ *   lane's snapshot doesn't cover: the lane's ranges from the snapshot's article on that the window did after it
+ *   (leased, done since the snapshot, or read past it) are planned again (the batch store's keys make the redo
+ *   idempotent); without a snapshot the window is lost and the group continues at its resume point (one seam).
+ * - next: the group's lowest planned range of the lane (or the one this node already has), leased to the node; none
+ *   when the lane is caught up: the worker snapshots and gives the group back, and gets it again when the tail
+ *   planned more.
+ * - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR; `<group>.gz`, the tail's
+ *   `<group>-tail.gz`), given to the next holder of the lane.
  */
 class GroupLeases {
   get dir(): string {
     return coordinatorConfig.snapshotDir
   }
 
-  /** LEASE_CAPTURE_ONLY: only the corpus windows' ranges (the history's first buckets share their priorities). */
-  private get captureOnly(): string {
-    return coordinatorConfig.leaseCaptureOnly ? 'AND r.capture' : ''
+  /**
+   * The lane's ranges, as a condition on `r`: the tail lane's are at or below the tail's priority, the history lane's
+   * above it — within the lease bounds when `bounds` (what is handed out), or all of them (what a lane's window did).
+   */
+  private laneFilter(lane: Lane, bounds: boolean): { sql: string; bindings: number[] } {
+    const tail = coordinatorConfig.tail.priority
+    if (lane === 'tail') return { sql: 'r.priority <= ?', bindings: [tail] }
+    if (!bounds) return { sql: 'r.priority > ?', bindings: [tail] }
+    const { leasePriorityMin: min, leasePriorityMax: max, leaseCaptureOnly } = coordinatorConfig
+    return {
+      sql: `r.priority > ? AND r.priority BETWEEN ? AND ?${leaseCaptureOnly ? ' AND r.capture' : ''}`,
+      bindings: [tail, min, max],
+    }
   }
 
-  async lease(nodeId: number): Promise<GroupLease | null> {
+  async lease(nodeId: number, lane: Lane = 'history'): Promise<GroupLease | null> {
+    // The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
+    if (lane === 'history' && (await CatalogSpace.full())) return null
     const rules = await RuleSets.current()
     const ttl = coordinatorConfig.leaseTtlSeconds
-    const { leasePriorityMin: min, leasePriorityMax: max } = coordinatorConfig
-    // The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
-    const history = (await CatalogSpace.full()) ? 'AND r.priority < 0' : ''
+    const todo = this.laneFilter(lane, true)
+    const all = this.laneFilter(lane, false)
     return db.transaction(async (trx) => {
       // One lease at a time: the choice reads several tables.
       await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
+      // The free groups with planned work in the lane (a range leased by a holder gone quiet counts: a group whose
+      // last range that holder took would never be taken over), by the most urgent range's priority; within a priority the
+      // most planned articles and the longest unheld alternate: one lease starts from the biggest, the next from
+      // the longest unheld (the sequence counts the leases; ranked afresh each time, "biggest first" would always
+      // win).
+      const { rows: turns } = await trx.rawQuery(`SELECT nextval('group_lease_turns') % 2 AS odd`)
+      const [first, second] = Number(turns[0].odd) === 1 ? [1, 0] : [0, 1]
       const { rows } = await trx.rawQuery(
-        `SELECT g.id, g.name, g.priority, l.node_id AS holder, l.snapshot_next, todo.priority AS todo_priority
-         FROM groups g
-         JOIN LATERAL (SELECT min(r.priority) AS priority FROM ranges r WHERE r.group_id = g.id AND r.status = 'planned' AND r.next_id <= r.end_id AND r.priority BETWEEN ? AND ? ${history} ${this.captureOnly}) todo ON true
-         LEFT JOIN group_leases l ON l.group_id = g.id
-         WHERE g.active AND todo.priority IS NOT NULL
-           AND (l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?))
-         ORDER BY todo.priority, l.heartbeat_at NULLS FIRST, g.priority
-         LIMIT 1`,
-        [min, max, ttl]
+        `WITH todo AS (
+           SELECT r.group_id, min(r.priority) AS priority, sum(r.end_id - r.next_id + 1) AS articles
+           FROM ranges r
+           WHERE (r.status = 'planned' OR (r.status = 'leased' AND r.heartbeat_at < now() - make_interval(secs => ?)))
+             AND r.next_id <= r.end_id AND ${todo.sql}
+           GROUP BY r.group_id
+         ), free AS (
+           SELECT g.id, g.name, g.priority AS group_priority, l.snapshot_next, l.snapshot_at, l.heartbeat_at,
+                  todo.priority, todo.articles
+           FROM todo JOIN groups g ON g.id = todo.group_id AND g.active
+           LEFT JOIN group_leases l ON l.group_id = g.id AND l.lane = ?
+           WHERE l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?)
+         ), turns AS (
+           SELECT free.*,
+                  least(2 * row_number() OVER (PARTITION BY priority ORDER BY articles DESC, group_priority, id) - ?,
+                        2 * row_number() OVER (PARTITION BY priority ORDER BY heartbeat_at NULLS FIRST, group_priority, id) - ?) AS turn
+           FROM free
+         )
+         SELECT * FROM turns ORDER BY priority, turn, group_priority, id LIMIT 1`,
+        [ttl, ...todo.bindings, lane, ttl, first, second]
       )
       const row = rows[0]
       if (!row) return null
       const groupId = Number(row.id)
-      // What the snapshot doesn't cover is read again: the ranges from its article on, the one holding it from there.
+      // What the snapshot doesn't cover is read again: the lane's ranges from its article on that the window did
+      // after it (a range done before the snapshot was taken — by the old range leases, say — stays done).
       if (row.snapshot_next !== null) {
         await trx.rawQuery(
-          `UPDATE ranges SET status = 'planned', node_id = NULL, next_id = greatest(start_id, ?::bigint)
-           WHERE group_id = ? AND end_id >= ?::bigint AND (status IN ('leased', 'done') OR next_id > ?::bigint)`,
-          [row.snapshot_next, groupId, row.snapshot_next, row.snapshot_next]
+          `UPDATE ranges r SET status = 'planned', node_id = NULL, next_id = greatest(r.start_id, ?::bigint)
+           WHERE r.group_id = ? AND r.end_id >= ?::bigint AND ${all.sql}
+             AND (r.status = 'leased' OR (r.status = 'done' AND (r.done_at IS NULL OR r.done_at >= ?)) OR (r.status = 'planned' AND r.next_id > ?::bigint))`,
+          [
+            row.snapshot_next,
+            groupId,
+            row.snapshot_next,
+            ...all.bindings,
+            row.snapshot_at ?? new Date(0),
+            row.snapshot_next,
+          ]
         )
       } else {
         await trx.rawQuery(
-          `UPDATE ranges SET status = 'planned', node_id = NULL WHERE group_id = ? AND status = 'leased'`,
-          [groupId]
+          `UPDATE ranges r SET status = 'planned', node_id = NULL WHERE r.group_id = ? AND r.status = 'leased' AND ${all.sql}`,
+          [groupId, ...all.bindings]
         )
       }
       await trx.rawQuery(
-        `INSERT INTO group_leases (group_id, node_id, leased_at, heartbeat_at, range_id)
-         VALUES (?, ?, now(), now(), NULL)
-         ON CONFLICT (group_id) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
-        [groupId, nodeId]
+        `INSERT INTO group_leases (group_id, lane, node_id, leased_at, heartbeat_at, range_id)
+         VALUES (?, ?, ?, now(), now(), NULL)
+         ON CONFLICT (group_id, lane) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
+        [groupId, lane, nodeId]
       )
       await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
       return {
-        group: { id: groupId, name: row.name, priority: Number(row.priority) },
+        group: { id: groupId, name: row.name, priority: Number(row.group_priority) },
+        lane,
         rules,
         snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
       }
     })
   }
 
-  /** The group's lease row, when this node holds it. */
-  async held(groupId: number, nodeId: number) {
-    const lease = await db.from('group_leases').where('group_id', groupId).first()
+  /** The group's lease row of the lane, when this node holds it. */
+  async held(groupId: number, nodeId: number, lane: Lane = 'history') {
+    const lease = await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .first()
     if (!lease || Number(lease.node_id) !== nodeId)
-      throw new LeaseLost(`Group ${groupId} is not held by this node`)
+      throw new LeaseLost(`Group ${groupId} (${lane}) is not held by this node`)
     return lease
   }
 
-  /** The group's next range in article order (or the one this node has), leased to the node; null when caught up. */
-  async next(groupId: number, nodeId: number): Promise<NextRange | null> {
-    await this.held(groupId, nodeId)
+  /**
+   * The group's next range of the lane in article order (or the one this node has), leased to the node; null when
+   * the lane is caught up.
+   */
+  async next(groupId: number, nodeId: number, lane: Lane = 'history'): Promise<NextRange | null> {
+    await this.held(groupId, nodeId, lane)
     const rules = await RuleSets.current()
-    const { leasePriorityMin: min, leasePriorityMax: max } = coordinatorConfig
-    const mine = await db
-      .from('ranges')
-      .where('group_id', groupId)
-      .where('status', 'leased')
-      .where('node_id', nodeId)
-      .orderBy('start_id')
-      .first()
+    const todo = this.laneFilter(lane, true)
+    const all = this.laneFilter(lane, false)
+    const mine = (
+      await db.rawQuery(
+        `SELECT r.id, r.start_id, r.end_id, r.next_id FROM ranges r
+         WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql} ORDER BY r.start_id LIMIT 1`,
+        [groupId, nodeId, ...all.bindings]
+      )
+    ).rows[0]
     const row =
       mine ??
       (
         await db.rawQuery(
           `UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
            WHERE r.id = (
-             SELECT id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND r.priority BETWEEN ? AND ? ${this.captureOnly}
+             SELECT r.id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND ${todo.sql}
              ORDER BY r.start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
            RETURNING r.id, r.start_id, r.end_id, r.next_id`,
-          [nodeId, rules.version, groupId, min, max]
+          [nodeId, rules.version, groupId, ...todo.bindings]
         )
       ).rows[0]
     await db
       .from('group_leases')
       .where('group_id', groupId)
+      .where('lane', lane)
       .update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
     if (!row) return null
     return {
@@ -134,63 +193,81 @@ class GroupLeases {
     }
   }
 
-  async heartbeat(groupId: number, nodeId: number) {
-    await this.held(groupId, nodeId)
-    await db.from('group_leases').where('group_id', groupId).update({ heartbeat_at: new Date() })
+  async heartbeat(groupId: number, nodeId: number, lane: Lane = 'history') {
+    await this.held(groupId, nodeId, lane)
+    await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', lane)
+      .update({ heartbeat_at: new Date() })
     await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
   }
 
-  /** A range of the group finished by its holder: counted on the lease (Leases.finish does the range). */
-  async rangeDone(groupId: number, headers: number) {
+  /** The lane a range belongs to, by its priority. */
+  laneOf(priority: number): Lane {
+    return priority <= coordinatorConfig.tail.priority ? 'tail' : 'history'
+  }
+
+  /** A range of the group finished by its holder: counted on the lane's lease (Leases.finish does the range). */
+  async rangeDone(groupId: number, headers: number, lane: Lane = 'history') {
     await db.rawQuery(
-      `UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ?`,
-      [headers, groupId]
+      `UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ? AND lane = ?`,
+      [headers, groupId, lane]
     )
   }
 
   /** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
-  async release(groupId: number, nodeId: number) {
-    await this.held(groupId, nodeId)
-    await db
-      .from('ranges')
-      .where('group_id', groupId)
-      .where('status', 'leased')
-      .where('node_id', nodeId)
-      .update({ status: 'planned', node_id: null })
+  async release(groupId: number, nodeId: number, lane: Lane = 'history') {
+    await this.held(groupId, nodeId, lane)
+    const all = this.laneFilter(lane, false)
+    await db.rawQuery(
+      `UPDATE ranges r SET status = 'planned', node_id = NULL
+       WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql}`,
+      [groupId, nodeId, ...all.bindings]
+    )
     await db
       .from('group_leases')
       .where('group_id', groupId)
+      .where('lane', lane)
       .update({ node_id: null, range_id: null })
   }
 
-  private snapshotPath(groupId: number) {
-    return join(this.dir, `${groupId}.gz`)
+  snapshotPath(groupId: number, lane: Lane = 'history') {
+    return join(this.dir, lane === 'tail' ? `${groupId}-tail.gz` : `${groupId}.gz`)
   }
 
   /**
    * The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
    * gigabytes) and written whole before it replaces the old one.
    */
-  async putSnapshot(groupId: number, nodeId: number, next: number, body: Readable) {
-    await this.held(groupId, nodeId)
+  async putSnapshot(
+    groupId: number,
+    nodeId: number,
+    next: number,
+    body: Readable,
+    lane: Lane = 'history'
+  ) {
+    await this.held(groupId, nodeId, lane)
     await mkdir(this.dir, { recursive: true })
-    const path = this.snapshotPath(groupId)
+    const path = this.snapshotPath(groupId, lane)
     await pipeline(body, createWriteStream(`${path}.part`))
     await rename(`${path}.part`, path)
     await db
       .from('group_leases')
       .where('group_id', groupId)
+      .where('lane', lane)
       .update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
   }
 
   /** The snapshot's stream and size, or null when there is none. */
   async getSnapshot(
     groupId: number,
-    nodeId: number
+    nodeId: number,
+    lane: Lane = 'history'
   ): Promise<{ stream: Readable; size: number; next: number } | null> {
-    const lease = await this.held(groupId, nodeId)
+    const lease = await this.held(groupId, nodeId, lane)
     if (lease.snapshot_next === null) return null
-    const path = this.snapshotPath(groupId)
+    const path = this.snapshotPath(groupId, lane)
     try {
       const { size } = await stat(path)
       return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }

app/services/prefetcher.ts

@@ -425,8 +425,9 @@ export class Prefetcher {
       )
     // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
     const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
-    // The groups workers hold (group_leases; REBUILD.md design 1): the prefetcher runs ahead of their cursors.
-    const held: number[] = (
+    // The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
+    // cursors (a group's history and tail lanes are two cursors).
+    const held: { group: number; lane: string }[] = (
       await db
         .from('group_leases')
         .whereNotNull('node_id')
@@ -435,8 +436,8 @@ export class Prefetcher {
           '>=',
           new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
         )
-        .select('group_id')
-    ).map((l) => Number(l.group_id))
+        .select('group_id', 'lane')
+    ).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
     // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
     const ahead = Math.max(
       coordinatorConfig.spool.minAhead,
@@ -460,22 +461,36 @@ export class Prefetcher {
     const want = Math.max(ahead - Number(ready), short)
     // The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
     const full = await CatalogSpace.full()
-    // Ahead of each held group's cursor: its next ranges in article order (within what the group leases hand out:
-    // LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY), a range of every held group before any group's second; without
-    // held groups (no window worker running) nothing is fetched ahead for workers.
+    // Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
+    // leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
+    // at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
+    // window worker running) nothing is fetched ahead for workers.
     const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
+    const tailPriority = coordinatorConfig.tail.priority
+    const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
+    const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
     const fresh =
-      room && want > 0 && held.length > 0
+      room && want > 0 && heldHistory.length + heldTail.length > 0
         ? await rows(db.query())
             .where('r.status', 'planned')
             .where('r.spooled_chunks', 0)
             .whereRaw('r.next_id <= r.end_id')
-            .whereIn('r.group_id', held)
-            .whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
-            .where((q: any) => (leaseCaptureOnly ? q.where('r.capture', true) : q))
-            .where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
+            .where((q: any) =>
+              q
+                .where((h: any) =>
+                  h
+                    .whereIn('r.group_id', heldHistory)
+                    .where('r.priority', '>', tailPriority)
+                    .whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
+                    .where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
+                )
+                .orWhere((t: any) =>
+                  t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
+                )
+            )
             .orderByRaw(
-              'row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id'
+              'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
+              [tailPriority]
             )
             .limit(want)
         : []

app/services/status.ts

@@ -0,0 +1,333 @@
+import db from '@adonisjs/lucid/services/db'
+import { readdir, stat, statfs } from 'node:fs/promises'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+import GroupLeases from '#services/group_leases'
+import type { Lane } from '#lib/edge/index'
+
+export interface RangeBand {
+  band: string
+  planned: number
+  plannedArticles: number
+  leased: number
+  done: number
+  doneArticles: number
+  failed: number
+}
+
+export interface HeldGroup {
+  node: string
+  group: string
+  lane: Lane
+  range: { start: number; end: number; next: number } | null
+  heartbeatSeconds: number
+  snapshotNext: number | null
+  snapshotSeconds: number | null
+  snapshotMb: number | null
+  rangesDone: number
+  headers: number
+}
+
+export interface SnapshotFile {
+  group: string
+  lane: Lane
+  mb: number
+  hours: number
+}
+
+export interface Status {
+  at: string
+  catalog: {
+    tablespace: string
+    gb: number
+    capGb: number
+    full: boolean
+    databaseGb: number
+    freeGb: number
+  }
+  bands: RangeBand[]
+  lastHour: {
+    indexedArticles: number
+    doneRanges: number
+    storedArticles: number
+    storedRanges: number
+  }
+  corpus: { pendingRanges: number; pendingArticles: number }
+  spool: { ranges: number; chunks: number }
+  tail: { groups: number; ranges: number; articles: number }
+  held: HeldGroup[]
+  snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
+  releases: { estimated: number; provisional: number }
+  probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
+  judge: { dirty: number }
+  nodes: { name: string; lastSeenSeconds: number | null }[]
+}
+
+const seconds = (since: Date | string | null) =>
+  since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
+
+/**
+ * What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
+ * NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
+ * with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
+ * filesystem; nothing is kept between calls.
+ */
+class StatusService {
+  async collect(): Promise<Status> {
+    const tail = coordinatorConfig.tail.priority
+    const cap = coordinatorConfig.catalogMaxNvmeGb
+    const { rows: space } = await db.rawQuery(
+      `SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
+              pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes,
+              pg_database_size(current_database()) AS database_bytes`
+    )
+    const tablespaceGb = Number(space[0].bytes) / 1e9
+    const fs = await statfs('/').catch(() => null)
+    const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
+
+    // The ranges by band: the tail, the capture windows by priority, the history by its thousand.
+    const { rows: bands } = await db.rawQuery(
+      `SELECT CASE WHEN priority <= ? THEN 'tail'
+                   WHEN priority < 0 THEN 'raised'
+                   WHEN capture THEN 'window ' || priority
+                   WHEN priority < 1000 THEN 'history 0-999'
+                   ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
+              count(*) FILTER (WHERE status = 'planned') AS planned,
+              coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
+              count(*) FILTER (WHERE status = 'leased') AS leased,
+              count(*) FILTER (WHERE status = 'done') AS done,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
+              count(*) FILTER (WHERE status = 'failed') AS failed
+       FROM ranges GROUP BY 1 ORDER BY min(priority)`,
+      [tail]
+    )
+    const { rows: hour } = await db.rawQuery(
+      `SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
+              count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
+              count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
+              count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
+              coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
+              count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
+              coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
+              count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
+              count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
+              coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles
+       FROM ranges`,
+      [tail, tail, tail]
+    )
+    const h = hour[0]
+
+    const ttl = coordinatorConfig.leaseTtlSeconds
+    const { rows: held } = await db.rawQuery(
+      `SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
+              r.start_id, r.end_id, r.next_id, l.group_id
+       FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
+       LEFT JOIN ranges r ON r.id = l.range_id
+       WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
+       ORDER BY l.lane, g.name`,
+      [ttl]
+    )
+    const snapshotSizes = new Map<string, number>()
+    const files: SnapshotFile[] = []
+    const names = new Map<number, string>(
+      (await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
+    )
+    for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
+      const m = /^(\d+)(-tail)?\.gz$/.exec(file)
+      if (!m) continue
+      const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
+      if (!s) continue
+      const lane: Lane = m[2] ? 'tail' : 'history'
+      snapshotSizes.set(`${m[1]}:${lane}`, s.size)
+      files.push({
+        group: names.get(Number(m[1])) ?? m[1],
+        lane,
+        mb: s.size / 1e6,
+        hours: (Date.now() - s.mtimeMs) / 3600_000,
+      })
+    }
+    files.sort((a, b) => b.mb - a.mb)
+
+    const { rows: releases } = await db.rawQuery(
+      `SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
+              (SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
+              (SELECT count(*) FROM probe_candidates WHERE status = 'waiting' AND ready) AS ready,
+              (SELECT count(*) FROM probe_candidates WHERE status = 'waiting') AS waiting,
+              (SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
+              (SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
+              (SELECT count(*) FROM judge_dirty) AS dirty`
+    )
+    const r = releases[0]
+    const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
+
+    return {
+      at: new Date().toISOString(),
+      catalog: {
+        tablespace: space[0].tablespace,
+        gb: tablespaceGb,
+        capGb: cap,
+        full: cap > 0 && tablespaceGb > cap,
+        databaseGb: Number(space[0].database_bytes) / 1e9,
+        freeGb,
+      },
+      bands: bands.map((b: any) => ({
+        band: b.band,
+        planned: Number(b.planned),
+        plannedArticles: Number(b.planned_articles),
+        leased: Number(b.leased),
+        done: Number(b.done),
+        doneArticles: Number(b.done_articles),
+        failed: Number(b.failed),
+      })),
+      lastHour: {
+        indexedArticles: Number(h.indexed),
+        doneRanges: Number(h.done_ranges),
+        storedArticles: Number(h.stored),
+        storedRanges: Number(h.stored_ranges),
+      },
+      corpus: {
+        pendingRanges: Number(h.corpus_pending),
+        pendingArticles: Number(h.corpus_pending_articles),
+      },
+      spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
+      tail: {
+        groups: Number(h.tail_groups),
+        ranges: Number(h.tail_ranges),
+        articles: Number(h.tail_articles),
+      },
+      held: held.map((l: any) => {
+        const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
+        return {
+          node: l.node,
+          group: l.group,
+          lane: l.lane,
+          range:
+            l.start_id !== null
+              ? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
+              : null,
+          heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
+          snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
+          snapshotSeconds: seconds(l.snapshot_at),
+          snapshotMb: size === undefined ? null : size / 1e6,
+          rangesDone: Number(l.ranges_done),
+          headers: Number(l.headers),
+        }
+      }),
+      snapshots: {
+        count: files.length,
+        gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
+        oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
+        largest: files.slice(0, 5),
+      },
+      releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
+      probes: {
+        ready: Number(r.ready),
+        waiting: Number(r.waiting),
+        pendingWaiting: Number(r.pending_waiting),
+        pendingNamed: Number(r.pending_named),
+      },
+      judge: { dirty: Number(r.dirty) },
+      nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
+    }
+  }
+
+  /** The status as a text panel. */
+  render(s: Status): string {
+    const n = (v: number) => Math.round(v).toLocaleString('en-US')
+    const b = (v: number) =>
+      v >= 1e9
+        ? `${(v / 1e9).toFixed(2)} B`
+        : v >= 1e6
+          ? `${(v / 1e6).toFixed(1)} M`
+          : v >= 1e3
+            ? `${(v / 1e3).toFixed(0)} k`
+            : String(v)
+    const age = (sec: number | null) =>
+      sec === null
+        ? '-'
+        : sec < 90
+          ? `${sec} s`
+          : sec < 5400
+            ? `${Math.round(sec / 60)} min`
+            : `${(sec / 3600).toFixed(1)} h`
+    const lines: string[] = []
+    lines.push(`Status at ${s.at}`)
+    lines.push('')
+    lines.push(
+      `Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; database ${s.catalog.databaseGb.toFixed(1)} GB; ${s.catalog.freeGb.toFixed(1)} GB free on /`
+    )
+    lines.push(
+      `Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
+    )
+    lines.push(
+      `Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups`
+    )
+    lines.push('')
+    const table = (header: string[], rows: string[][]) => {
+      const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
+      const line = (r: string[]) =>
+        r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join('  ')
+      lines.push(line(header))
+      for (const r of rows) lines.push(line(r))
+    }
+    lines.push('Ranges by band:')
+    table(
+      ['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
+      s.bands.map((x) => [
+        x.band,
+        n(x.planned),
+        b(x.plannedArticles),
+        n(x.leased),
+        n(x.done),
+        b(x.doneArticles),
+        n(x.failed),
+      ])
+    )
+    lines.push('')
+    lines.push(`Held groups (${s.held.length}):`)
+    if (s.held.length > 0)
+      table(
+        [
+          'group',
+          'lane',
+          'node',
+          'range',
+          'at',
+          'heartbeat',
+          'snapshot',
+          'size',
+          'done',
+          'headers',
+        ],
+        s.held.map((x) => [
+          x.group,
+          x.lane,
+          x.node,
+          x.range ? `${x.range.start}-${x.range.end}` : '-',
+          x.range ? String(x.range.next) : '-',
+          age(x.heartbeatSeconds),
+          x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
+          x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
+          n(x.rangesDone),
+          b(x.headers),
+        ])
+      )
+    lines.push('')
+    lines.push(
+      `Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
+    )
+    lines.push(
+      `Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
+    )
+    lines.push(
+      `Probes: ${n(s.probes.ready)} ready of ${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
+    )
+    lines.push(
+      `Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
+    )
+    return lines.join('\n')
+  }
+}
+
+export default new StatusService()