Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end

A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail
planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades
through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads`
(EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The
lease order within a lane alternates the group with the most planned articles and the longest unheld one. The
takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range
is held by a holder gone quiet can be taken over.

`coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with
their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the
corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot
file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/status.spec.ts

@@ -0,0 +1,157 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+import StatusService from '#services/status'
+
+test.group('Status', (group) => {
+  let dir = ''
+  let snapshotDir = ''
+  group.each.setup(async () => {
+    const rollback = await testUtils.db().withGlobalTransaction()
+    dir = await mkdtemp(join(tmpdir(), 'snapshots-'))
+    snapshotDir = coordinatorConfig.snapshotDir
+    coordinatorConfig.snapshotDir = dir
+    return rollback
+  })
+  group.each.teardown(async () => {
+    coordinatorConfig.snapshotDir = snapshotDir
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  test('the ranges by band, the held groups with their snapshots, the backlogs', async ({
+    assert,
+  }) => {
+    const [g] = await db
+      .table('groups')
+      .insert({ name: 'alt.binaries.status', priority: 1 })
+      .returning('id')
+    const [n] = await db
+      .table('nodes')
+      .insert({ name: 'status-node', last_seen_at: new Date() })
+      .returning('id')
+    const ranges = [
+      { start_id: 1, end_id: 100, next_id: 1, priority: -3 },
+      { start_id: 101, end_id: 200, next_id: 101, priority: -1 },
+      {
+        start_id: 201,
+        end_id: 300,
+        next_id: 301,
+        priority: 0,
+        capture: true,
+        status: 'done',
+        done_at: new Date(),
+      },
+      {
+        start_id: 301,
+        end_id: 400,
+        next_id: 351,
+        priority: 0,
+        capture: true,
+        status: 'leased',
+        node_id: n.id,
+        spooled_chunks: 2,
+      },
+      {
+        start_id: 401,
+        end_id: 500,
+        next_id: 401,
+        priority: 2,
+        capture: true,
+        stored_at: new Date(),
+      },
+      { start_id: 501, end_id: 600, next_id: 501, priority: 500 },
+      { start_id: 601, end_id: 700, next_id: 601, priority: 4200, status: 'failed' },
+    ]
+    const ids = await db
+      .table('ranges')
+      .multiInsert(ranges.map((r) => ({ group_id: g.id, ...r })))
+      .returning('id')
+    await db.table('group_leases').multiInsert([
+      {
+        group_id: g.id,
+        lane: 'history',
+        node_id: n.id,
+        leased_at: new Date(),
+        heartbeat_at: new Date(),
+        range_id: ids[3].id,
+        snapshot_next: 301,
+        snapshot_at: new Date(Date.now() - 600_000),
+        ranges_done: 1,
+        headers: 100,
+      },
+      {
+        group_id: g.id,
+        lane: 'tail',
+        node_id: null,
+        heartbeat_at: new Date(Date.now() - 3600_000),
+        snapshot_next: 1,
+        snapshot_at: new Date(Date.now() - 3600_000),
+      },
+    ])
+    await writeFile(join(dir, `${g.id}.gz`), Buffer.alloc(2_000_000))
+    await writeFile(join(dir, `${g.id}-tail.gz`), Buffer.alloc(1000))
+
+    const s = await StatusService.collect()
+    assert.deepEqual(
+      s.bands.map((b) => [
+        b.band,
+        b.planned,
+        b.plannedArticles,
+        b.leased,
+        b.done,
+        b.doneArticles,
+        b.failed,
+      ]),
+      [
+        ['tail', 1, 100, 0, 0, 0, 0],
+        ['raised', 1, 100, 0, 0, 0, 0],
+        ['window 0', 0, 0, 1, 1, 100, 0],
+        ['window 2', 1, 100, 0, 0, 0, 0],
+        ['history 0-999', 1, 100, 0, 0, 0, 0],
+        ['history 4000+', 0, 0, 0, 0, 0, 1],
+      ]
+    )
+    assert.deepEqual(s.lastHour, {
+      indexedArticles: 100,
+      doneRanges: 1,
+      storedArticles: 100,
+      storedRanges: 1,
+    })
+    assert.deepEqual(s.corpus, { pendingRanges: 1, pendingArticles: 100 })
+    assert.deepEqual(s.spool, { ranges: 1, chunks: 2 })
+    assert.deepEqual(s.tail, { groups: 1, ranges: 1, articles: 100 })
+    assert.lengthOf(s.held, 1)
+    assert.include(s.held[0], {
+      node: 'status-node',
+      group: 'alt.binaries.status',
+      lane: 'history',
+      snapshotNext: 301,
+      snapshotMb: 2,
+      rangesDone: 1,
+      headers: 100,
+    })
+    assert.deepEqual(s.held[0].range, { start: 301, end: 400, next: 351 })
+    assert.isAtLeast(s.held[0].snapshotSeconds!, 599)
+    assert.equal(s.snapshots.count, 2)
+    assert.deepEqual(
+      s.snapshots.largest.map((f) => [f.group, f.lane, f.mb]),
+      [
+        ['alt.binaries.status', 'history', 2],
+        ['alt.binaries.status', 'tail', 0.001],
+      ]
+    )
+    assert.equal(s.releases.provisional, 0)
+    assert.deepEqual(s.probes, { ready: 0, waiting: 0, pendingWaiting: 0, pendingNamed: 0 })
+    assert.equal(s.nodes.find((x) => x.name === 'status-node')?.lastSeenSeconds, 0)
+    assert.isAbove(s.catalog.gb, 0)
+
+    const text = StatusService.render(s)
+    assert.include(text, 'window 0')
+    assert.include(text, 'alt.binaries.status  history  status-node  301-400  351')
+    assert.include(text, 'Snapshots: 2 files')
+  })
+})