Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end

A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail
planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades
through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads`
(EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The
lease order within a lane alternates the group with the most planned articles and the longest unheld one. The
takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range
is held by a holder gone quiet can be taken over.

`coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with
their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the
corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot
file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/e2e/edge_node.spec.ts

@@ -0,0 +1,238 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import Node from '#models/node'
+import Spool from '#services/spool'
+import Corpus from '#services/corpus'
+import GroupLeases from '#services/group_leases'
+import coordinatorConfig from '#config/coordinator'
+import { GROUP, overviewChunk, runEdge, startEdge, waitFor } from '#tests/e2e/helpers/edge'
+
+/**
+ * The Rust edge worker against the real node API, database, spool and corpus (REBUILD.md, "Voyager 3: window
+ * indexing", 1 and 5): what broke in production on 2026-10-27 (LOG.md (118)) — a stream waiting on a chunk the
+ * corpus fill stored, a failed stream, a restart with and without the snapshot, a takeover.
+ */
+test.group('Edge worker end to end', (group) => {
+  const saved = {
+    spool: coordinatorConfig.spool.dir,
+    corpus: coordinatorConfig.corpus.dir,
+    snapshots: coordinatorConfig.snapshotDir,
+    chunk: coordinatorConfig.chunkArticles,
+    wait: coordinatorConfig.streamChunkWaitMs,
+  }
+  let dir = ''
+  group.each.setup(async () => {
+    dir = await mkdtemp(join(tmpdir(), 'e2e-'))
+    coordinatorConfig.spool.dir = join(dir, 'spool')
+    coordinatorConfig.corpus.dir = join(dir, 'corpus')
+    coordinatorConfig.snapshotDir = join(dir, 'snapshots')
+    coordinatorConfig.chunkArticles = 100
+    return testUtils.db().truncate()
+  })
+  group.each.teardown(async () => {
+    Object.assign(coordinatorConfig.spool, { dir: saved.spool })
+    coordinatorConfig.corpus.dir = saved.corpus
+    coordinatorConfig.snapshotDir = saved.snapshots
+    coordinatorConfig.chunkArticles = saved.chunk
+    coordinatorConfig.streamChunkWaitMs = saved.wait
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  /** A group with one range of three chunks and a node with a token. */
+  async function setup(opts: { capture?: boolean } = {}) {
+    const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
+    const [r] = await db
+      .table('ranges')
+      .insert({
+        group_id: g.id,
+        start_id: 1,
+        end_id: 300,
+        next_id: 1,
+        priority: 0,
+        capture: opts.capture ?? false,
+      })
+      .returning('id')
+    const node = await Node.create({ name: 'e2e-node' })
+    const token = (await Node.accessTokens.create(node)).value!.release()
+    return { groupId: Number(g.id), rangeId: Number(r.id), nodeId: Number(node.id), token }
+  }
+  const range = async (id: number) => (await db.from('ranges').where('id', id).firstOrFail()) as any
+  const lease = async (groupId: number) =>
+    (await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', 'history')
+      .first()) as any
+  /** The release whose first article is `at` (a release's name isn't the file's: the identity strips it). */
+  const releaseAt = async (at: number) =>
+    (await db.from('releases').where('first_article', at).first()) as any
+  const names = async () =>
+    JSON.stringify(
+      await db
+        .from('releases')
+        .select('name', 'first_article', 'last_article', 'segments', 'complete')
+        .orderBy('first_article')
+    )
+  const has = (lines: string[], text: string) => lines.some((l) => l.includes(text))
+
+  test('a range streamed from the spool and the corpus, with a chunk the corpus fill stores while the stream waits', async ({
+    assert,
+  }) => {
+    const { groupId, rangeId, token } = await setup({ capture: true })
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const run = await startEdge(token)
+    // The worker holds the group and streams: chunk 1 goes out, chunk 2 is waited for — the corpus fill stores it.
+    await waitFor(
+      'the group to be held',
+      async () =>
+        (await lease(groupId))?.range_id !== null && (await lease(groupId))?.range_id !== undefined
+    )
+    await new Promise((r) => setTimeout(r, 700))
+    await Corpus.write(
+      { group: GROUP, start: 1, end: 300 },
+      { from: 101, to: 200 },
+      overviewChunk(101, 200),
+      3
+    )
+    const { code, lines } = { ...(await run.exit), lines: run.lines }
+    assert.equal(code, 0, lines.join('\n'))
+    assert.isTrue(has(lines, 'caught up after 1 ranges'), lines.join('\n'))
+    assert.isTrue(has(lines, 'snapshot at 301'), lines.join('\n'))
+    assert.include(await range(rangeId), { status: 'done' })
+    assert.equal(Number((await range(rangeId)).headers), 300)
+    const l = await lease(groupId)
+    assert.isNull(l.node_id)
+    assert.equal(Number(l.snapshot_next), 301)
+    assert.equal(Number(l.ranges_done), 1)
+    // The window closed on the early files (the frontier is a day past them); every closed release is settled.
+    const releases = await db
+      .from('releases')
+      .select('name', 'settled_at', 'first_article', 'last_article', 'segments')
+    assert.isAtLeast(releases.length, 10)
+    assert.isTrue(releases.every((r) => r.settled_at !== null))
+    const spanning = releases.find((r) => Number(r.first_article) === 100)
+    assert.isDefined(spanning, await names())
+    assert.deepEqual(
+      [Number(spanning.first_article), Number(spanning.last_article), Number(spanning.segments)],
+      [100, 101, 2],
+      'the file over the chunk boundary came through the carried window'
+    )
+  })
+
+  test('a stream that fails: the batch so far is posted, the window snapshotted, the group given back; the next run restores it', async ({
+    assert,
+  }) => {
+    coordinatorConfig.streamChunkWaitMs = 1500
+    const { groupId, rangeId, token } = await setup()
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    const first = await runEdge(token)
+    assert.equal(first.code, 0, first.lines.join('\n'))
+    assert.isTrue(has(first.lines, 'the group goes back'), first.lines.join('\n'))
+    assert.isTrue(has(first.lines, 'snapshot at 101'), first.lines.join('\n'))
+    let r = await range(rangeId)
+    assert.include(r, { status: 'planned', node_id: null })
+    assert.equal(
+      Number(r.next_id),
+      101,
+      'the batch for chunk 1 was stored before the group went back'
+    )
+    let l = await lease(groupId)
+    assert.isNull(l.node_id)
+    assert.equal(Number(l.snapshot_next), 101)
+    const before = await db.from('releases').count('* as n').first()
+    assert.isAtLeast(Number(before!.n), 10)
+    assert.isNull(await releaseAt(100), 'the file over the boundary is still open in the window')
+
+    // The next run: the window restored, the rest of the range streamed, the open file completed.
+    coordinatorConfig.streamChunkWaitMs = saved.wait
+    await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const second = await runEdge(token)
+    assert.equal(second.code, 0, second.lines.join('\n'))
+    assert.isTrue(
+      has(second.lines, 'window restored from the snapshot at 101'),
+      second.lines.join('\n')
+    )
+    r = await range(rangeId)
+    assert.equal(r.status, 'done')
+    assert.equal(Number(r.headers), 300)
+    l = await lease(groupId)
+    assert.equal(Number(l.snapshot_next), 301)
+    const spanning = await releaseAt(100)
+    assert.isNotNull(spanning, await names())
+    assert.deepEqual(
+      [Number(spanning.first_article), Number(spanning.last_article), Number(spanning.segments)],
+      [100, 101, 2]
+    )
+  })
+
+  test('a restart whose snapshot file is gone: the window starts empty and the file over the seam ends incomplete', async ({
+    assert,
+  }) => {
+    coordinatorConfig.streamChunkWaitMs = 1500
+    const { groupId, rangeId, token } = await setup()
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    const first = await runEdge(token)
+    assert.isTrue(has(first.lines, 'snapshot at 101'), first.lines.join('\n'))
+    await rm(GroupLeases.snapshotPath(groupId), { force: true })
+
+    coordinatorConfig.streamChunkWaitMs = saved.wait
+    await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const second = await runEdge(token)
+    assert.equal(second.code, 0, second.lines.join('\n'))
+    assert.isTrue(
+      has(second.lines, 'but there is none; the window starts empty'),
+      second.lines.join('\n')
+    )
+    assert.equal((await range(rangeId)).status, 'done')
+    // Part 1 was in the lost window: one seam. Part 2 alone is a file short of a part.
+    assert.isNull(await releaseAt(100), await names())
+    const half = await releaseAt(101)
+    assert.isNotNull(half, await names())
+    assert.deepEqual([Number(half.segments), half.complete], [1, false], await names())
+  })
+
+  test("a takeover: the stale holder's next batch is refused, its snapshot too, and the group stays the new holder's", async ({
+    assert,
+  }) => {
+    const { groupId, rangeId, token, nodeId } = await setup()
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    const run = await startEdge(token)
+    // Chunk 1 posted; the stream now waits for chunk 2.
+    await waitFor('the first batch', async () => Number((await range(rangeId)).next_id) === 101)
+    // The holder looks dead (its group and range heartbeats stale): another node takes the group over (no snapshot
+    // yet: the range goes back to its resume point).
+    const stale = new Date(Date.now() - 24 * 3600_000)
+    await db.from('group_leases').where('group_id', groupId).update({ heartbeat_at: stale })
+    await db.from('ranges').where('id', rangeId).update({ heartbeat_at: stale })
+    const other = await Node.create({ name: 'e2e-other' })
+    const taken = await GroupLeases.lease(Number(other.id))
+    assert.equal(taken?.group.id, groupId)
+    assert.isNull(taken?.snapshot)
+    assert.include(await range(rangeId), { status: 'planned', node_id: null })
+    // The old holder gets chunk 2 all the same, posts, and is refused.
+    await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const { code } = await run.exit
+    const lines = run.lines
+    assert.equal(code, 0, lines.join('\n'))
+    assert.isTrue(has(lines, 'snapshot failed'), lines.join('\n'))
+    assert.isTrue(
+      has(lines, 'not leased to this node') || has(lines, 'no longer this node'),
+      lines.join('\n')
+    )
+    const l = await lease(groupId)
+    assert.equal(Number(l.node_id), Number(other.id))
+    assert.isNull(l.snapshot_next)
+    const r = await range(rangeId)
+    assert.equal(r.status, 'planned')
+    assert.equal(Number(r.next_id), 101, 'nothing past the first batch was stored')
+    assert.notEqual(Number(r.node_id), nodeId)
+  })
+})

tests/e2e/helpers/edge.ts

@@ -0,0 +1,149 @@
+import { spawn } from 'node:child_process'
+import { access } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { deflateSync } from 'node:zlib'
+import env from '#start/env'
+
+export const GROUP = 'alt.binaries.e2e'
+
+/** The edge worker binary: EDGE_NODE_BIN, else the release or debug build under edge/target. */
+export async function edgeBinary(): Promise<string> {
+  const candidates = [
+    process.env.EDGE_NODE_BIN,
+    'edge/target/release/edge-node',
+    'edge/target/debug/edge-node',
+  ]
+    .filter((c): c is string => !!c)
+    .map((c) => resolve(c))
+  for (const path of candidates) {
+    try {
+      await access(path)
+      return path
+    } catch {}
+  }
+  throw new Error(
+    `No edge-node binary (looked at ${candidates.join(', ')}): build it with "cargo build --release --manifest-path edge/Cargo.toml --bin edge-node", or set EDGE_NODE_BIN`
+  )
+}
+
+/** The test server's address for the worker (tests/bootstrap.ts starts it on HOST:PORT). */
+export function coordinatorUrl(): string {
+  const host = env.get('HOST', 'localhost')
+  return `http://${host === '0.0.0.0' ? '127.0.0.1' : host}:${env.get('PORT', 3333)}`
+}
+
+export interface EdgeRun {
+  exit: Promise<{ code: number | null; signal: string | null }>
+  lines: string[]
+  kill: () => void
+}
+
+/**
+ * The real edge worker against the test server: one thread on the history lane, one group, a batch per chunk
+ * (100 articles in these tests), a snapshot after every range.
+ */
+export async function startEdge(token: string, args: string[] = []): Promise<EdgeRun> {
+  const bin = await edgeBinary()
+  // The test server first (tests/bootstrap.ts starts it for this suite): a worker that can't connect retries for
+  // minutes before it says so.
+  const probe = await fetch(`${coordinatorUrl()}/api/v1/node/lease`, { method: 'POST' }).catch(
+    (e) => e as Error
+  )
+  if (probe instanceof Error || probe.status !== 401)
+    throw new Error(
+      `The test server doesn't answer at ${coordinatorUrl()}: ${probe instanceof Error ? probe.message : `HTTP ${probe.status}`}`
+    )
+  const child = spawn(
+    bin,
+    [
+      '--threads',
+      '1',
+      '--tail-threads',
+      '0',
+      '--groups',
+      '1',
+      '--batch-articles',
+      '100',
+      '--snapshot-minutes',
+      '0',
+      ...args,
+    ],
+    {
+      env: { ...process.env, COORDINATOR_URL: coordinatorUrl(), NODE_TOKEN: token },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    }
+  )
+  const lines: string[] = []
+  let rest = ''
+  const collect = (chunk: Buffer) => {
+    rest += chunk.toString('utf8')
+    const parts = rest.split('\n')
+    rest = parts.pop() ?? ''
+    lines.push(...parts)
+  }
+  child.stdout.on('data', collect)
+  child.stderr.on('data', collect)
+  const exit = new Promise<{ code: number | null; signal: string | null }>((done) => {
+    child.on('exit', (code, signal) => {
+      if (rest) lines.push(rest)
+      done({ code, signal })
+    })
+  })
+  return { exit, lines, kill: () => child.kill('SIGKILL') }
+}
+
+/** Runs the worker to its end (at most `timeoutMs`), its log lines and exit code. */
+export async function runEdge(token: string, args: string[] = [], timeoutMs = 60_000) {
+  const run = await startEdge(token, args)
+  const timer = setTimeout(() => run.kill(), timeoutMs)
+  const result = await run.exit
+  clearTimeout(timer)
+  return { ...result, lines: run.lines }
+}
+
+/** Polls `condition` every 100 ms until it holds or `timeoutMs` passed (then throws). */
+export async function waitFor(what: string, condition: () => Promise<boolean>, timeoutMs = 30_000) {
+  const until = Date.now() + timeoutMs
+  while (Date.now() < until) {
+    if (await condition()) return
+    await new Promise((r) => setTimeout(r, 100))
+  }
+  throw new Error(`Timed out waiting for ${what}`)
+}
+
+const BASE_MS = Date.UTC(2026, 9, 5, 10)
+
+/** A file name without digits (a number sequence would make an album of a poster's files): letters from `k`. */
+export function nameOf(k: number): string {
+  let s = ''
+  let n = k
+  do {
+    s = String.fromCharCode(97 + (n % 26)) + s
+    n = Math.floor(n / 26)
+  } while (n > 0)
+  return `Set.${s}.mkv`
+}
+
+/**
+ * The overview lines of articles `from`..`to`, one an hour from 2026-10-05: article 1 is a one-part file, and from 2
+ * on each even article is part 1 of a two-part file whose part 2 is the next (odd) article — so a file spans every
+ * chunk boundary at an even `to` (100 and 101, 200 and 201: the window has to carry it). Dates an hour apart make
+ * the posting frontier move a day every 24 articles, so the window closes on files as it goes.
+ */
+export function overviewLines(from: number, to: number): string[] {
+  const lines: string[] = []
+  for (let n = from; n <= to; n++) {
+    const date = new Date(BASE_MS + n * 3600_000).toUTCString().replace(' GMT', ' +0000')
+    const [name, part] =
+      n === 1 ? [nameOf(1), '(1/1)'] : n % 2 === 0 ? [nameOf(n), '(1/2)'] : [nameOf(n - 1), '(2/2)']
+    lines.push(
+      `${n}\t"${name}" yEnc ${part}\tposter <p@x>\t${date}\t<a${n}@e2e>\t\t700000\t5000\tXref: news.example ${GROUP}:${n}`
+    )
+  }
+  return lines
+}
+
+/** One XOVER chunk as the news server sends it (deflated). */
+export function overviewChunk(from: number, to: number): Buffer {
+  return deflateSync(Buffer.from(overviewLines(from, to).join('\r\n') + '\r\n'))
+}

tests/functional/group_leases.spec.ts

@@ -42,7 +42,7 @@ test.group('Group leases', (g) => {
     await rm(dir, { recursive: true, force: true })
   })
 
-  test('a lease takes the group with the most urgent planned range; its ranges come in article order; caught up it gives nothing', async ({
+  test('a lease takes the group with the most urgent planned range of its lane; its ranges come in article order; caught up it gives nothing', async ({
     assert,
   }) => {
     const a = await group('alt.binaries.gl.a', 1)
@@ -56,33 +56,73 @@ test.group('Group leases', (g) => {
     const n1 = await node('gl-1')
     const n2 = await node('gl-2')
 
+    // The history lane: both groups have priority 0 work; a has the most planned articles (300 to 100).
     const first = await GroupLeases.lease(n1)
-    assert.equal(first?.group.id, b, 'the tail first')
+    assert.equal(first?.group.id, a, 'the most planned articles first')
+    assert.equal(first?.lane, 'history')
     assert.isNull(first?.snapshot)
     // Held: the next node gets the other group; a third gets nothing.
     const second = await GroupLeases.lease(n2)
-    assert.equal(second?.group.id, a)
+    assert.equal(second?.group.id, b)
     assert.isNull(await GroupLeases.lease(await node('gl-3')))
 
-    // Ranges in article order, whatever their priority.
-    assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b1, start: 1, end: 100, next: 1 })
-    assert.include(await rangeRow(b1), { status: 'leased', node_id: n1 })
+    // Ranges in article order, whatever their priority; the tail range isn't the history lane's.
+    assert.deepEqual((await GroupLeases.next(b, n2))?.range, { id: b2, start: 101, end: 200, next: 101 })
+    assert.include(await rangeRow(b2), { status: 'leased', node_id: n2 })
     // Asked again before finishing: the same range.
-    assert.equal((await GroupLeases.next(b, n1))?.range.id, b1)
-    await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
-    assert.equal((await GroupLeases.next(b, n1))?.range.id, b2)
+    assert.equal((await GroupLeases.next(b, n2))?.range.id, b2)
     // A chunk of b2 that doesn't inflate, reported after the batch before it: the range is asked for again and
     // comes back at its resume point (the chunk fetched plain next time).
     await db.from('ranges').where('id', b2).update({ next_id: 151 })
-    await Leases.corrupt(b2, n1, 151)
+    await Leases.corrupt(b2, n2, 151)
     assert.include(await rangeRow(b2), { status: 'planned', node_id: null })
-    assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b2, start: 101, end: 200, next: 151 })
+    assert.deepEqual((await GroupLeases.next(b, n2))?.range, { id: b2, start: 101, end: 200, next: 151 })
     assert.deepEqual((await rangeRow(b2)).plain_chunks.map(Number), [151])
     await db.from('ranges').where('id', b2).update({ status: 'done', next_id: 201 })
-    assert.isNull(await GroupLeases.next(b, n1), 'caught up')
+    assert.isNull(await GroupLeases.next(b, n2), 'caught up: the tail range is the other lane\'s')
+    assert.include(await rangeRow(b1), { status: 'planned' })
+
+    // The tail lane: group b's tail, held by a third node while n2 holds b's history.
+    const n3 = await node('gl-3b')
+    const tail = await GroupLeases.lease(n3, 'tail')
+    assert.equal(tail?.group.id, b)
+    assert.equal(tail?.lane, 'tail')
+    assert.isNull(await GroupLeases.lease(await node('gl-3c'), 'tail'), 'no other group has tail work')
+    assert.deepEqual((await GroupLeases.next(b, n3, 'tail'))?.range, { id: b1, start: 1, end: 100, next: 1 })
+    // The history lane is n2's.
+    await assert.rejects(() => GroupLeases.next(b, n3), LeaseLost)
+    await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
+    assert.isNull(await GroupLeases.next(b, n3, 'tail'), 'the tail lane is caught up')
+    // Each lane its own snapshot.
+    await GroupLeases.putSnapshot(b, n3, 101, Readable.from([Buffer.from('tail at 101')]), 'tail')
+    await GroupLeases.putSnapshot(b, n2, 201, Readable.from([Buffer.from('history at 201')]))
+    assert.equal((await GroupLeases.getSnapshot(b, n3, 'tail'))?.next, 101)
+    assert.equal((await GroupLeases.getSnapshot(b, n2))?.next, 201)
+    await GroupLeases.release(b, n3, 'tail')
+    assert.include(await GroupLeases.held(b, n2), { node_id: n2 })
+    await assert.rejects(() => GroupLeases.held(b, n3, 'tail'), LeaseLost)
 
     assert.deepEqual([a1, a2, a3].length, 3)
-    await assert.rejects(() => GroupLeases.next(a, n1), LeaseLost)
+    await assert.rejects(() => GroupLeases.next(a, n2), LeaseLost)
+  })
+
+  test('the order mixes the group with the most planned articles and the longest unheld one', async ({ assert }) => {
+    // Four groups with priority 0 work: c was held before (5000 articles left), a, b and d never (100, 1000, 50).
+    const a = await group('alt.binaries.gl.o.a', 1)
+    const b = await group('alt.binaries.gl.o.b', 2)
+    const c = await group('alt.binaries.gl.o.c', 3)
+    const d = await group('alt.binaries.gl.o.d', 4)
+    await range(a, 1, 0)
+    for (let i = 0; i < 10; i++) await range(b, 1 + 100 * i, 0)
+    for (let i = 0; i < 50; i++) await range(c, 1 + 100 * i, 0)
+    await db.table('ranges').insert({ group_id: d, start_id: 1, end_id: 50, next_id: 1, priority: 0 })
+    await db.table('group_leases').insert({ group_id: c, lane: 'history', node_id: null, heartbeat_at: new Date(Date.now() - 3600_000) })
+    // By articles c, b, a, d; by age a, b, d (never held, by group priority), c. The leases alternate which
+    // ranking goes first: c, a, b, d (or a, c, b, d when the sequence stands on the other parity).
+    await db.rawQuery(`SELECT setval('group_lease_turns', 2)`)
+    const order = []
+    for (const name of ['o1', 'o2', 'o3', 'o4']) order.push((await GroupLeases.lease(await node(name)))?.group.id)
+    assert.deepEqual(order, [c, a, b, d])
   })
 
   test('a stale holder is taken over and the ranges after the snapshot are read again; a release keeps the resume point', async ({
@@ -165,14 +205,19 @@ test.group('Group leases', (g) => {
     const r2 = await range(a, 101, 0)
     const n1 = await node('gl-6')
     await GroupLeases.lease(n1)
+    // A range after the snapshot's article done long before the snapshot was taken (the old range leases did it):
+    // not the window's, it stays done.
+    const r3 = await range(a, 201, 0)
+    await db.from('ranges').where('id', r3).update({ status: 'done', next_id: 301, done_at: new Date(Date.now() - 48 * 3600_000) })
     // The worker snapshotted at the start, finished r1, and crashed before its next snapshot: r1 is read again.
     await GroupLeases.putSnapshot(a, n1, 1, Readable.from([Buffer.from('window at 1')]))
-    await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101 })
+    await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101, done_at: new Date() })
     await db.from('group_leases').where('group_id', a).update({ heartbeat_at: new Date(Date.now() - 24 * 3600_000) })
     await GroupLeases.lease(await node('gl-7'))
     assert.equal((await rangeRow(r1)).status, 'planned')
     assert.equal(Number((await rangeRow(r1)).next_id), 1)
     assert.equal((await rangeRow(r2)).status, 'planned')
     assert.equal(Number((await rangeRow(r2)).next_id), 101)
+    assert.equal((await rangeRow(r3)).status, 'done')
   })
 })

tests/functional/node_api.spec.ts

@@ -145,6 +145,23 @@ test.group('Node API', (group) => {
       ;(await client.post(`/api/v1/node/groups/${groupId}/release`).bearerToken(token)).assertStatus(200)
       ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
       assert.include(await db.from('ranges').where('id', r2.id).firstOrFail(), { status: 'planned', node_id: null })
+
+      // The tail lane (`?lane=tail`): the group's tail range, with a cursor and snapshot of its own.
+      const [t] = await db.table('ranges').insert({ group_id: g.id, start_id: 401, end_id: 500, next_id: 401, priority: -3 }).returning('id')
+      ;(await client.post('/api/v1/node/groups/lease?lane=sideways').bearerToken(token)).assertStatus(400)
+      const tail = await client.post('/api/v1/node/groups/lease?lane=tail').bearerToken(token)
+      tail.assertStatus(200)
+      assert.equal(tail.body().lane, 'tail')
+      assert.equal(Number(tail.body().group.id), groupId)
+      assert.deepEqual((await client.post(`/api/v1/node/groups/${groupId}/next?lane=tail`).bearerToken(token)).body().range, { id: Number(t.id), start: 401, end: 500, next: 401 })
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat?lane=tail`).bearerToken(token)).assertStatus(200)
+      ;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot?next=401&lane=tail`, Buffer.from('tail at 401'))).assertStatus(200)
+      const gotTail = await client.get(`/api/v1/node/groups/${groupId}/snapshot?lane=tail`).bearerToken(token)
+      gotTail.assertStatus(200)
+      assert.equal(gotTail.header('x-snapshot-next'), '401')
+      ;(await client.post(`/api/v1/node/groups/${groupId}/release?lane=tail`).bearerToken(token)).assertStatus(200)
+      assert.include(await db.from('ranges').where('id', t.id).firstOrFail(), { status: 'planned', node_id: null })
     } finally {
       coordinatorConfig.corpus.dir = saved.corpus
       coordinatorConfig.snapshotDir = saved.snapshots

tests/functional/status.spec.ts

@@ -0,0 +1,157 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+import StatusService from '#services/status'
+
+test.group('Status', (group) => {
+  let dir = ''
+  let snapshotDir = ''
+  group.each.setup(async () => {
+    const rollback = await testUtils.db().withGlobalTransaction()
+    dir = await mkdtemp(join(tmpdir(), 'snapshots-'))
+    snapshotDir = coordinatorConfig.snapshotDir
+    coordinatorConfig.snapshotDir = dir
+    return rollback
+  })
+  group.each.teardown(async () => {
+    coordinatorConfig.snapshotDir = snapshotDir
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  test('the ranges by band, the held groups with their snapshots, the backlogs', async ({
+    assert,
+  }) => {
+    const [g] = await db
+      .table('groups')
+      .insert({ name: 'alt.binaries.status', priority: 1 })
+      .returning('id')
+    const [n] = await db
+      .table('nodes')
+      .insert({ name: 'status-node', last_seen_at: new Date() })
+      .returning('id')
+    const ranges = [
+      { start_id: 1, end_id: 100, next_id: 1, priority: -3 },
+      { start_id: 101, end_id: 200, next_id: 101, priority: -1 },
+      {
+        start_id: 201,
+        end_id: 300,
+        next_id: 301,
+        priority: 0,
+        capture: true,
+        status: 'done',
+        done_at: new Date(),
+      },
+      {
+        start_id: 301,
+        end_id: 400,
+        next_id: 351,
+        priority: 0,
+        capture: true,
+        status: 'leased',
+        node_id: n.id,
+        spooled_chunks: 2,
+      },
+      {
+        start_id: 401,
+        end_id: 500,
+        next_id: 401,
+        priority: 2,
+        capture: true,
+        stored_at: new Date(),
+      },
+      { start_id: 501, end_id: 600, next_id: 501, priority: 500 },
+      { start_id: 601, end_id: 700, next_id: 601, priority: 4200, status: 'failed' },
+    ]
+    const ids = await db
+      .table('ranges')
+      .multiInsert(ranges.map((r) => ({ group_id: g.id, ...r })))
+      .returning('id')
+    await db.table('group_leases').multiInsert([
+      {
+        group_id: g.id,
+        lane: 'history',
+        node_id: n.id,
+        leased_at: new Date(),
+        heartbeat_at: new Date(),
+        range_id: ids[3].id,
+        snapshot_next: 301,
+        snapshot_at: new Date(Date.now() - 600_000),
+        ranges_done: 1,
+        headers: 100,
+      },
+      {
+        group_id: g.id,
+        lane: 'tail',
+        node_id: null,
+        heartbeat_at: new Date(Date.now() - 3600_000),
+        snapshot_next: 1,
+        snapshot_at: new Date(Date.now() - 3600_000),
+      },
+    ])
+    await writeFile(join(dir, `${g.id}.gz`), Buffer.alloc(2_000_000))
+    await writeFile(join(dir, `${g.id}-tail.gz`), Buffer.alloc(1000))
+
+    const s = await StatusService.collect()
+    assert.deepEqual(
+      s.bands.map((b) => [
+        b.band,
+        b.planned,
+        b.plannedArticles,
+        b.leased,
+        b.done,
+        b.doneArticles,
+        b.failed,
+      ]),
+      [
+        ['tail', 1, 100, 0, 0, 0, 0],
+        ['raised', 1, 100, 0, 0, 0, 0],
+        ['window 0', 0, 0, 1, 1, 100, 0],
+        ['window 2', 1, 100, 0, 0, 0, 0],
+        ['history 0-999', 1, 100, 0, 0, 0, 0],
+        ['history 4000+', 0, 0, 0, 0, 0, 1],
+      ]
+    )
+    assert.deepEqual(s.lastHour, {
+      indexedArticles: 100,
+      doneRanges: 1,
+      storedArticles: 100,
+      storedRanges: 1,
+    })
+    assert.deepEqual(s.corpus, { pendingRanges: 1, pendingArticles: 100 })
+    assert.deepEqual(s.spool, { ranges: 1, chunks: 2 })
+    assert.deepEqual(s.tail, { groups: 1, ranges: 1, articles: 100 })
+    assert.lengthOf(s.held, 1)
+    assert.include(s.held[0], {
+      node: 'status-node',
+      group: 'alt.binaries.status',
+      lane: 'history',
+      snapshotNext: 301,
+      snapshotMb: 2,
+      rangesDone: 1,
+      headers: 100,
+    })
+    assert.deepEqual(s.held[0].range, { start: 301, end: 400, next: 351 })
+    assert.isAtLeast(s.held[0].snapshotSeconds!, 599)
+    assert.equal(s.snapshots.count, 2)
+    assert.deepEqual(
+      s.snapshots.largest.map((f) => [f.group, f.lane, f.mb]),
+      [
+        ['alt.binaries.status', 'history', 2],
+        ['alt.binaries.status', 'tail', 0.001],
+      ]
+    )
+    assert.equal(s.releases.provisional, 0)
+    assert.deepEqual(s.probes, { ready: 0, waiting: 0, pendingWaiting: 0, pendingNamed: 0 })
+    assert.equal(s.nodes.find((x) => x.name === 'status-node')?.lastSeenSeconds, 0)
+    assert.isAbove(s.catalog.gb, 0)
+
+    const text = StatusService.render(s)
+    assert.include(text, 'window 0')
+    assert.include(text, 'alt.binaries.status  history  status-node  301-400  351')
+    assert.include(text, 'Snapshots: 2 files')
+  })
+})