Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end

A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail
planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades
through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads`
(EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The
lease order within a lane alternates the group with the most planned articles and the longest unheld one. The
takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range
is held by a holder gone quiet can be taken over.

`coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with
their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the
corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot
file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/group_leases.spec.ts

@@ -42,7 +42,7 @@ test.group('Group leases', (g) => {
     await rm(dir, { recursive: true, force: true })
   })
 
-  test('a lease takes the group with the most urgent planned range; its ranges come in article order; caught up it gives nothing', async ({
+  test('a lease takes the group with the most urgent planned range of its lane; its ranges come in article order; caught up it gives nothing', async ({
     assert,
   }) => {
     const a = await group('alt.binaries.gl.a', 1)
@@ -56,33 +56,73 @@ test.group('Group leases', (g) => {
     const n1 = await node('gl-1')
     const n2 = await node('gl-2')
 
+    // The history lane: both groups have priority 0 work; a has the most planned articles (300 to 100).
     const first = await GroupLeases.lease(n1)
-    assert.equal(first?.group.id, b, 'the tail first')
+    assert.equal(first?.group.id, a, 'the most planned articles first')
+    assert.equal(first?.lane, 'history')
     assert.isNull(first?.snapshot)
     // Held: the next node gets the other group; a third gets nothing.
     const second = await GroupLeases.lease(n2)
-    assert.equal(second?.group.id, a)
+    assert.equal(second?.group.id, b)
     assert.isNull(await GroupLeases.lease(await node('gl-3')))
 
-    // Ranges in article order, whatever their priority.
-    assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b1, start: 1, end: 100, next: 1 })
-    assert.include(await rangeRow(b1), { status: 'leased', node_id: n1 })
+    // Ranges in article order, whatever their priority; the tail range isn't the history lane's.
+    assert.deepEqual((await GroupLeases.next(b, n2))?.range, { id: b2, start: 101, end: 200, next: 101 })
+    assert.include(await rangeRow(b2), { status: 'leased', node_id: n2 })
     // Asked again before finishing: the same range.
-    assert.equal((await GroupLeases.next(b, n1))?.range.id, b1)
-    await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
-    assert.equal((await GroupLeases.next(b, n1))?.range.id, b2)
+    assert.equal((await GroupLeases.next(b, n2))?.range.id, b2)
     // A chunk of b2 that doesn't inflate, reported after the batch before it: the range is asked for again and
     // comes back at its resume point (the chunk fetched plain next time).
     await db.from('ranges').where('id', b2).update({ next_id: 151 })
-    await Leases.corrupt(b2, n1, 151)
+    await Leases.corrupt(b2, n2, 151)
     assert.include(await rangeRow(b2), { status: 'planned', node_id: null })
-    assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b2, start: 101, end: 200, next: 151 })
+    assert.deepEqual((await GroupLeases.next(b, n2))?.range, { id: b2, start: 101, end: 200, next: 151 })
     assert.deepEqual((await rangeRow(b2)).plain_chunks.map(Number), [151])
     await db.from('ranges').where('id', b2).update({ status: 'done', next_id: 201 })
-    assert.isNull(await GroupLeases.next(b, n1), 'caught up')
+    assert.isNull(await GroupLeases.next(b, n2), 'caught up: the tail range is the other lane\'s')
+    assert.include(await rangeRow(b1), { status: 'planned' })
+
+    // The tail lane: group b's tail, held by a third node while n2 holds b's history.
+    const n3 = await node('gl-3b')
+    const tail = await GroupLeases.lease(n3, 'tail')
+    assert.equal(tail?.group.id, b)
+    assert.equal(tail?.lane, 'tail')
+    assert.isNull(await GroupLeases.lease(await node('gl-3c'), 'tail'), 'no other group has tail work')
+    assert.deepEqual((await GroupLeases.next(b, n3, 'tail'))?.range, { id: b1, start: 1, end: 100, next: 1 })
+    // The history lane is n2's.
+    await assert.rejects(() => GroupLeases.next(b, n3), LeaseLost)
+    await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
+    assert.isNull(await GroupLeases.next(b, n3, 'tail'), 'the tail lane is caught up')
+    // Each lane its own snapshot.
+    await GroupLeases.putSnapshot(b, n3, 101, Readable.from([Buffer.from('tail at 101')]), 'tail')
+    await GroupLeases.putSnapshot(b, n2, 201, Readable.from([Buffer.from('history at 201')]))
+    assert.equal((await GroupLeases.getSnapshot(b, n3, 'tail'))?.next, 101)
+    assert.equal((await GroupLeases.getSnapshot(b, n2))?.next, 201)
+    await GroupLeases.release(b, n3, 'tail')
+    assert.include(await GroupLeases.held(b, n2), { node_id: n2 })
+    await assert.rejects(() => GroupLeases.held(b, n3, 'tail'), LeaseLost)
 
     assert.deepEqual([a1, a2, a3].length, 3)
-    await assert.rejects(() => GroupLeases.next(a, n1), LeaseLost)
+    await assert.rejects(() => GroupLeases.next(a, n2), LeaseLost)
+  })
+
+  test('the order mixes the group with the most planned articles and the longest unheld one', async ({ assert }) => {
+    // Four groups with priority 0 work: c was held before (5000 articles left), a, b and d never (100, 1000, 50).
+    const a = await group('alt.binaries.gl.o.a', 1)
+    const b = await group('alt.binaries.gl.o.b', 2)
+    const c = await group('alt.binaries.gl.o.c', 3)
+    const d = await group('alt.binaries.gl.o.d', 4)
+    await range(a, 1, 0)
+    for (let i = 0; i < 10; i++) await range(b, 1 + 100 * i, 0)
+    for (let i = 0; i < 50; i++) await range(c, 1 + 100 * i, 0)
+    await db.table('ranges').insert({ group_id: d, start_id: 1, end_id: 50, next_id: 1, priority: 0 })
+    await db.table('group_leases').insert({ group_id: c, lane: 'history', node_id: null, heartbeat_at: new Date(Date.now() - 3600_000) })
+    // By articles c, b, a, d; by age a, b, d (never held, by group priority), c. The leases alternate which
+    // ranking goes first: c, a, b, d (or a, c, b, d when the sequence stands on the other parity).
+    await db.rawQuery(`SELECT setval('group_lease_turns', 2)`)
+    const order = []
+    for (const name of ['o1', 'o2', 'o3', 'o4']) order.push((await GroupLeases.lease(await node(name)))?.group.id)
+    assert.deepEqual(order, [c, a, b, d])
   })
 
   test('a stale holder is taken over and the ranges after the snapshot are read again; a release keeps the resume point', async ({
@@ -165,14 +205,19 @@ test.group('Group leases', (g) => {
     const r2 = await range(a, 101, 0)
     const n1 = await node('gl-6')
     await GroupLeases.lease(n1)
+    // A range after the snapshot's article done long before the snapshot was taken (the old range leases did it):
+    // not the window's, it stays done.
+    const r3 = await range(a, 201, 0)
+    await db.from('ranges').where('id', r3).update({ status: 'done', next_id: 301, done_at: new Date(Date.now() - 48 * 3600_000) })
     // The worker snapshotted at the start, finished r1, and crashed before its next snapshot: r1 is read again.
     await GroupLeases.putSnapshot(a, n1, 1, Readable.from([Buffer.from('window at 1')]))
-    await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101 })
+    await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101, done_at: new Date() })
     await db.from('group_leases').where('group_id', a).update({ heartbeat_at: new Date(Date.now() - 24 * 3600_000) })
     await GroupLeases.lease(await node('gl-7'))
     assert.equal((await rangeRow(r1)).status, 'planned')
     assert.equal(Number((await rangeRow(r1)).next_id), 1)
     assert.equal((await rangeRow(r2)).status, 'planned')
     assert.equal(Number((await rangeRow(r2)).next_id), 101)
+    assert.equal((await rangeRow(r3)).status, 'done')
   })
 })

tests/functional/node_api.spec.ts

@@ -145,6 +145,23 @@ test.group('Node API', (group) => {
       ;(await client.post(`/api/v1/node/groups/${groupId}/release`).bearerToken(token)).assertStatus(200)
       ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
       assert.include(await db.from('ranges').where('id', r2.id).firstOrFail(), { status: 'planned', node_id: null })
+
+      // The tail lane (`?lane=tail`): the group's tail range, with a cursor and snapshot of its own.
+      const [t] = await db.table('ranges').insert({ group_id: g.id, start_id: 401, end_id: 500, next_id: 401, priority: -3 }).returning('id')
+      ;(await client.post('/api/v1/node/groups/lease?lane=sideways').bearerToken(token)).assertStatus(400)
+      const tail = await client.post('/api/v1/node/groups/lease?lane=tail').bearerToken(token)
+      tail.assertStatus(200)
+      assert.equal(tail.body().lane, 'tail')
+      assert.equal(Number(tail.body().group.id), groupId)
+      assert.deepEqual((await client.post(`/api/v1/node/groups/${groupId}/next?lane=tail`).bearerToken(token)).body().range, { id: Number(t.id), start: 401, end: 500, next: 401 })
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat?lane=tail`).bearerToken(token)).assertStatus(200)
+      ;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot?next=401&lane=tail`, Buffer.from('tail at 401'))).assertStatus(200)
+      const gotTail = await client.get(`/api/v1/node/groups/${groupId}/snapshot?lane=tail`).bearerToken(token)
+      gotTail.assertStatus(200)
+      assert.equal(gotTail.header('x-snapshot-next'), '401')
+      ;(await client.post(`/api/v1/node/groups/${groupId}/release?lane=tail`).bearerToken(token)).assertStatus(200)
+      assert.include(await db.from('ranges').where('id', t.id).firstOrFail(), { status: 'planned', node_id: null })
     } finally {
       coordinatorConfig.corpus.dir = saved.corpus
       coordinatorConfig.snapshotDir = saved.snapshots

tests/functional/status.spec.ts

@@ -0,0 +1,157 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+import StatusService from '#services/status'
+
+test.group('Status', (group) => {
+  let dir = ''
+  let snapshotDir = ''
+  group.each.setup(async () => {
+    const rollback = await testUtils.db().withGlobalTransaction()
+    dir = await mkdtemp(join(tmpdir(), 'snapshots-'))
+    snapshotDir = coordinatorConfig.snapshotDir
+    coordinatorConfig.snapshotDir = dir
+    return rollback
+  })
+  group.each.teardown(async () => {
+    coordinatorConfig.snapshotDir = snapshotDir
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  test('the ranges by band, the held groups with their snapshots, the backlogs', async ({
+    assert,
+  }) => {
+    const [g] = await db
+      .table('groups')
+      .insert({ name: 'alt.binaries.status', priority: 1 })
+      .returning('id')
+    const [n] = await db
+      .table('nodes')
+      .insert({ name: 'status-node', last_seen_at: new Date() })
+      .returning('id')
+    const ranges = [
+      { start_id: 1, end_id: 100, next_id: 1, priority: -3 },
+      { start_id: 101, end_id: 200, next_id: 101, priority: -1 },
+      {
+        start_id: 201,
+        end_id: 300,
+        next_id: 301,
+        priority: 0,
+        capture: true,
+        status: 'done',
+        done_at: new Date(),
+      },
+      {
+        start_id: 301,
+        end_id: 400,
+        next_id: 351,
+        priority: 0,
+        capture: true,
+        status: 'leased',
+        node_id: n.id,
+        spooled_chunks: 2,
+      },
+      {
+        start_id: 401,
+        end_id: 500,
+        next_id: 401,
+        priority: 2,
+        capture: true,
+        stored_at: new Date(),
+      },
+      { start_id: 501, end_id: 600, next_id: 501, priority: 500 },
+      { start_id: 601, end_id: 700, next_id: 601, priority: 4200, status: 'failed' },
+    ]
+    const ids = await db
+      .table('ranges')
+      .multiInsert(ranges.map((r) => ({ group_id: g.id, ...r })))
+      .returning('id')
+    await db.table('group_leases').multiInsert([
+      {
+        group_id: g.id,
+        lane: 'history',
+        node_id: n.id,
+        leased_at: new Date(),
+        heartbeat_at: new Date(),
+        range_id: ids[3].id,
+        snapshot_next: 301,
+        snapshot_at: new Date(Date.now() - 600_000),
+        ranges_done: 1,
+        headers: 100,
+      },
+      {
+        group_id: g.id,
+        lane: 'tail',
+        node_id: null,
+        heartbeat_at: new Date(Date.now() - 3600_000),
+        snapshot_next: 1,
+        snapshot_at: new Date(Date.now() - 3600_000),
+      },
+    ])
+    await writeFile(join(dir, `${g.id}.gz`), Buffer.alloc(2_000_000))
+    await writeFile(join(dir, `${g.id}-tail.gz`), Buffer.alloc(1000))
+
+    const s = await StatusService.collect()
+    assert.deepEqual(
+      s.bands.map((b) => [
+        b.band,
+        b.planned,
+        b.plannedArticles,
+        b.leased,
+        b.done,
+        b.doneArticles,
+        b.failed,
+      ]),
+      [
+        ['tail', 1, 100, 0, 0, 0, 0],
+        ['raised', 1, 100, 0, 0, 0, 0],
+        ['window 0', 0, 0, 1, 1, 100, 0],
+        ['window 2', 1, 100, 0, 0, 0, 0],
+        ['history 0-999', 1, 100, 0, 0, 0, 0],
+        ['history 4000+', 0, 0, 0, 0, 0, 1],
+      ]
+    )
+    assert.deepEqual(s.lastHour, {
+      indexedArticles: 100,
+      doneRanges: 1,
+      storedArticles: 100,
+      storedRanges: 1,
+    })
+    assert.deepEqual(s.corpus, { pendingRanges: 1, pendingArticles: 100 })
+    assert.deepEqual(s.spool, { ranges: 1, chunks: 2 })
+    assert.deepEqual(s.tail, { groups: 1, ranges: 1, articles: 100 })
+    assert.lengthOf(s.held, 1)
+    assert.include(s.held[0], {
+      node: 'status-node',
+      group: 'alt.binaries.status',
+      lane: 'history',
+      snapshotNext: 301,
+      snapshotMb: 2,
+      rangesDone: 1,
+      headers: 100,
+    })
+    assert.deepEqual(s.held[0].range, { start: 301, end: 400, next: 351 })
+    assert.isAtLeast(s.held[0].snapshotSeconds!, 599)
+    assert.equal(s.snapshots.count, 2)
+    assert.deepEqual(
+      s.snapshots.largest.map((f) => [f.group, f.lane, f.mb]),
+      [
+        ['alt.binaries.status', 'history', 2],
+        ['alt.binaries.status', 'tail', 0.001],
+      ]
+    )
+    assert.equal(s.releases.provisional, 0)
+    assert.deepEqual(s.probes, { ready: 0, waiting: 0, pendingWaiting: 0, pendingNamed: 0 })
+    assert.equal(s.nodes.find((x) => x.name === 'status-node')?.lastSeenSeconds, 0)
+    assert.isAbove(s.catalog.gb, 0)
+
+    const text = StatusService.render(s)
+    assert.include(text, 'window 0')
+    assert.include(text, 'alt.binaries.status  history  status-node  301-400  351')
+    assert.include(text, 'Snapshots: 2 files')
+  })
+})