Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end

A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail
planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades
through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads`
(EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The
lease order within a lane alternates the group with the most planned articles and the longest unheld one. The
takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range
is held by a holder gone quiet can be taken over.

`coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with
their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the
corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot
file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/e2e/edge_node.spec.ts

@@ -0,0 +1,238 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import Node from '#models/node'
+import Spool from '#services/spool'
+import Corpus from '#services/corpus'
+import GroupLeases from '#services/group_leases'
+import coordinatorConfig from '#config/coordinator'
+import { GROUP, overviewChunk, runEdge, startEdge, waitFor } from '#tests/e2e/helpers/edge'
+
+/**
+ * The Rust edge worker against the real node API, database, spool and corpus (REBUILD.md, "Voyager 3: window
+ * indexing", 1 and 5): what broke in production on 2026-10-27 (LOG.md (118)) — a stream waiting on a chunk the
+ * corpus fill stored, a failed stream, a restart with and without the snapshot, a takeover.
+ */
+test.group('Edge worker end to end', (group) => {
+  const saved = {
+    spool: coordinatorConfig.spool.dir,
+    corpus: coordinatorConfig.corpus.dir,
+    snapshots: coordinatorConfig.snapshotDir,
+    chunk: coordinatorConfig.chunkArticles,
+    wait: coordinatorConfig.streamChunkWaitMs,
+  }
+  let dir = ''
+  group.each.setup(async () => {
+    dir = await mkdtemp(join(tmpdir(), 'e2e-'))
+    coordinatorConfig.spool.dir = join(dir, 'spool')
+    coordinatorConfig.corpus.dir = join(dir, 'corpus')
+    coordinatorConfig.snapshotDir = join(dir, 'snapshots')
+    coordinatorConfig.chunkArticles = 100
+    return testUtils.db().truncate()
+  })
+  group.each.teardown(async () => {
+    Object.assign(coordinatorConfig.spool, { dir: saved.spool })
+    coordinatorConfig.corpus.dir = saved.corpus
+    coordinatorConfig.snapshotDir = saved.snapshots
+    coordinatorConfig.chunkArticles = saved.chunk
+    coordinatorConfig.streamChunkWaitMs = saved.wait
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  /** A group with one range of three chunks and a node with a token. */
+  async function setup(opts: { capture?: boolean } = {}) {
+    const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
+    const [r] = await db
+      .table('ranges')
+      .insert({
+        group_id: g.id,
+        start_id: 1,
+        end_id: 300,
+        next_id: 1,
+        priority: 0,
+        capture: opts.capture ?? false,
+      })
+      .returning('id')
+    const node = await Node.create({ name: 'e2e-node' })
+    const token = (await Node.accessTokens.create(node)).value!.release()
+    return { groupId: Number(g.id), rangeId: Number(r.id), nodeId: Number(node.id), token }
+  }
+  const range = async (id: number) => (await db.from('ranges').where('id', id).firstOrFail()) as any
+  const lease = async (groupId: number) =>
+    (await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .where('lane', 'history')
+      .first()) as any
+  /** The release whose first article is `at` (a release's name isn't the file's: the identity strips it). */
+  const releaseAt = async (at: number) =>
+    (await db.from('releases').where('first_article', at).first()) as any
+  const names = async () =>
+    JSON.stringify(
+      await db
+        .from('releases')
+        .select('name', 'first_article', 'last_article', 'segments', 'complete')
+        .orderBy('first_article')
+    )
+  const has = (lines: string[], text: string) => lines.some((l) => l.includes(text))
+
+  test('a range streamed from the spool and the corpus, with a chunk the corpus fill stores while the stream waits', async ({
+    assert,
+  }) => {
+    const { groupId, rangeId, token } = await setup({ capture: true })
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const run = await startEdge(token)
+    // The worker holds the group and streams: chunk 1 goes out, chunk 2 is waited for — the corpus fill stores it.
+    await waitFor(
+      'the group to be held',
+      async () =>
+        (await lease(groupId))?.range_id !== null && (await lease(groupId))?.range_id !== undefined
+    )
+    await new Promise((r) => setTimeout(r, 700))
+    await Corpus.write(
+      { group: GROUP, start: 1, end: 300 },
+      { from: 101, to: 200 },
+      overviewChunk(101, 200),
+      3
+    )
+    const { code, lines } = { ...(await run.exit), lines: run.lines }
+    assert.equal(code, 0, lines.join('\n'))
+    assert.isTrue(has(lines, 'caught up after 1 ranges'), lines.join('\n'))
+    assert.isTrue(has(lines, 'snapshot at 301'), lines.join('\n'))
+    assert.include(await range(rangeId), { status: 'done' })
+    assert.equal(Number((await range(rangeId)).headers), 300)
+    const l = await lease(groupId)
+    assert.isNull(l.node_id)
+    assert.equal(Number(l.snapshot_next), 301)
+    assert.equal(Number(l.ranges_done), 1)
+    // The window closed on the early files (the frontier is a day past them); every closed release is settled.
+    const releases = await db
+      .from('releases')
+      .select('name', 'settled_at', 'first_article', 'last_article', 'segments')
+    assert.isAtLeast(releases.length, 10)
+    assert.isTrue(releases.every((r) => r.settled_at !== null))
+    const spanning = releases.find((r) => Number(r.first_article) === 100)
+    assert.isDefined(spanning, await names())
+    assert.deepEqual(
+      [Number(spanning.first_article), Number(spanning.last_article), Number(spanning.segments)],
+      [100, 101, 2],
+      'the file over the chunk boundary came through the carried window'
+    )
+  })
+
+  test('a stream that fails: the batch so far is posted, the window snapshotted, the group given back; the next run restores it', async ({
+    assert,
+  }) => {
+    coordinatorConfig.streamChunkWaitMs = 1500
+    const { groupId, rangeId, token } = await setup()
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    const first = await runEdge(token)
+    assert.equal(first.code, 0, first.lines.join('\n'))
+    assert.isTrue(has(first.lines, 'the group goes back'), first.lines.join('\n'))
+    assert.isTrue(has(first.lines, 'snapshot at 101'), first.lines.join('\n'))
+    let r = await range(rangeId)
+    assert.include(r, { status: 'planned', node_id: null })
+    assert.equal(
+      Number(r.next_id),
+      101,
+      'the batch for chunk 1 was stored before the group went back'
+    )
+    let l = await lease(groupId)
+    assert.isNull(l.node_id)
+    assert.equal(Number(l.snapshot_next), 101)
+    const before = await db.from('releases').count('* as n').first()
+    assert.isAtLeast(Number(before!.n), 10)
+    assert.isNull(await releaseAt(100), 'the file over the boundary is still open in the window')
+
+    // The next run: the window restored, the rest of the range streamed, the open file completed.
+    coordinatorConfig.streamChunkWaitMs = saved.wait
+    await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const second = await runEdge(token)
+    assert.equal(second.code, 0, second.lines.join('\n'))
+    assert.isTrue(
+      has(second.lines, 'window restored from the snapshot at 101'),
+      second.lines.join('\n')
+    )
+    r = await range(rangeId)
+    assert.equal(r.status, 'done')
+    assert.equal(Number(r.headers), 300)
+    l = await lease(groupId)
+    assert.equal(Number(l.snapshot_next), 301)
+    const spanning = await releaseAt(100)
+    assert.isNotNull(spanning, await names())
+    assert.deepEqual(
+      [Number(spanning.first_article), Number(spanning.last_article), Number(spanning.segments)],
+      [100, 101, 2]
+    )
+  })
+
+  test('a restart whose snapshot file is gone: the window starts empty and the file over the seam ends incomplete', async ({
+    assert,
+  }) => {
+    coordinatorConfig.streamChunkWaitMs = 1500
+    const { groupId, rangeId, token } = await setup()
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    const first = await runEdge(token)
+    assert.isTrue(has(first.lines, 'snapshot at 101'), first.lines.join('\n'))
+    await rm(GroupLeases.snapshotPath(groupId), { force: true })
+
+    coordinatorConfig.streamChunkWaitMs = saved.wait
+    await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const second = await runEdge(token)
+    assert.equal(second.code, 0, second.lines.join('\n'))
+    assert.isTrue(
+      has(second.lines, 'but there is none; the window starts empty'),
+      second.lines.join('\n')
+    )
+    assert.equal((await range(rangeId)).status, 'done')
+    // Part 1 was in the lost window: one seam. Part 2 alone is a file short of a part.
+    assert.isNull(await releaseAt(100), await names())
+    const half = await releaseAt(101)
+    assert.isNotNull(half, await names())
+    assert.deepEqual([Number(half.segments), half.complete], [1, false], await names())
+  })
+
+  test("a takeover: the stale holder's next batch is refused, its snapshot too, and the group stays the new holder's", async ({
+    assert,
+  }) => {
+    const { groupId, rangeId, token, nodeId } = await setup()
+    await Spool.write(rangeId, { from: 1, to: 100 }, overviewChunk(1, 100))
+    const run = await startEdge(token)
+    // Chunk 1 posted; the stream now waits for chunk 2.
+    await waitFor('the first batch', async () => Number((await range(rangeId)).next_id) === 101)
+    // The holder looks dead (its group and range heartbeats stale): another node takes the group over (no snapshot
+    // yet: the range goes back to its resume point).
+    const stale = new Date(Date.now() - 24 * 3600_000)
+    await db.from('group_leases').where('group_id', groupId).update({ heartbeat_at: stale })
+    await db.from('ranges').where('id', rangeId).update({ heartbeat_at: stale })
+    const other = await Node.create({ name: 'e2e-other' })
+    const taken = await GroupLeases.lease(Number(other.id))
+    assert.equal(taken?.group.id, groupId)
+    assert.isNull(taken?.snapshot)
+    assert.include(await range(rangeId), { status: 'planned', node_id: null })
+    // The old holder gets chunk 2 all the same, posts, and is refused.
+    await Spool.write(rangeId, { from: 101, to: 200 }, overviewChunk(101, 200))
+    await Spool.write(rangeId, { from: 201, to: 300 }, overviewChunk(201, 300))
+    const { code } = await run.exit
+    const lines = run.lines
+    assert.equal(code, 0, lines.join('\n'))
+    assert.isTrue(has(lines, 'snapshot failed'), lines.join('\n'))
+    assert.isTrue(
+      has(lines, 'not leased to this node') || has(lines, 'no longer this node'),
+      lines.join('\n')
+    )
+    const l = await lease(groupId)
+    assert.equal(Number(l.node_id), Number(other.id))
+    assert.isNull(l.snapshot_next)
+    const r = await range(rangeId)
+    assert.equal(r.status, 'planned')
+    assert.equal(Number(r.next_id), 101, 'nothing past the first batch was stored')
+    assert.notEqual(Number(r.node_id), nodeId)
+  })
+})

tests/e2e/helpers/edge.ts

@@ -0,0 +1,149 @@
+import { spawn } from 'node:child_process'
+import { access } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { deflateSync } from 'node:zlib'
+import env from '#start/env'
+
+export const GROUP = 'alt.binaries.e2e'
+
+/** The edge worker binary: EDGE_NODE_BIN, else the release or debug build under edge/target. */
+export async function edgeBinary(): Promise<string> {
+  const candidates = [
+    process.env.EDGE_NODE_BIN,
+    'edge/target/release/edge-node',
+    'edge/target/debug/edge-node',
+  ]
+    .filter((c): c is string => !!c)
+    .map((c) => resolve(c))
+  for (const path of candidates) {
+    try {
+      await access(path)
+      return path
+    } catch {}
+  }
+  throw new Error(
+    `No edge-node binary (looked at ${candidates.join(', ')}): build it with "cargo build --release --manifest-path edge/Cargo.toml --bin edge-node", or set EDGE_NODE_BIN`
+  )
+}
+
+/** The test server's address for the worker (tests/bootstrap.ts starts it on HOST:PORT). */
+export function coordinatorUrl(): string {
+  const host = env.get('HOST', 'localhost')
+  return `http://${host === '0.0.0.0' ? '127.0.0.1' : host}:${env.get('PORT', 3333)}`
+}
+
+export interface EdgeRun {
+  exit: Promise<{ code: number | null; signal: string | null }>
+  lines: string[]
+  kill: () => void
+}
+
+/**
+ * The real edge worker against the test server: one thread on the history lane, one group, a batch per chunk
+ * (100 articles in these tests), a snapshot after every range.
+ */
+export async function startEdge(token: string, args: string[] = []): Promise<EdgeRun> {
+  const bin = await edgeBinary()
+  // The test server first (tests/bootstrap.ts starts it for this suite): a worker that can't connect retries for
+  // minutes before it says so.
+  const probe = await fetch(`${coordinatorUrl()}/api/v1/node/lease`, { method: 'POST' }).catch(
+    (e) => e as Error
+  )
+  if (probe instanceof Error || probe.status !== 401)
+    throw new Error(
+      `The test server doesn't answer at ${coordinatorUrl()}: ${probe instanceof Error ? probe.message : `HTTP ${probe.status}`}`
+    )
+  const child = spawn(
+    bin,
+    [
+      '--threads',
+      '1',
+      '--tail-threads',
+      '0',
+      '--groups',
+      '1',
+      '--batch-articles',
+      '100',
+      '--snapshot-minutes',
+      '0',
+      ...args,
+    ],
+    {
+      env: { ...process.env, COORDINATOR_URL: coordinatorUrl(), NODE_TOKEN: token },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    }
+  )
+  const lines: string[] = []
+  let rest = ''
+  const collect = (chunk: Buffer) => {
+    rest += chunk.toString('utf8')
+    const parts = rest.split('\n')
+    rest = parts.pop() ?? ''
+    lines.push(...parts)
+  }
+  child.stdout.on('data', collect)
+  child.stderr.on('data', collect)
+  const exit = new Promise<{ code: number | null; signal: string | null }>((done) => {
+    child.on('exit', (code, signal) => {
+      if (rest) lines.push(rest)
+      done({ code, signal })
+    })
+  })
+  return { exit, lines, kill: () => child.kill('SIGKILL') }
+}
+
+/** Runs the worker to its end (at most `timeoutMs`), its log lines and exit code. */
+export async function runEdge(token: string, args: string[] = [], timeoutMs = 60_000) {
+  const run = await startEdge(token, args)
+  const timer = setTimeout(() => run.kill(), timeoutMs)
+  const result = await run.exit
+  clearTimeout(timer)
+  return { ...result, lines: run.lines }
+}
+
+/** Polls `condition` every 100 ms until it holds or `timeoutMs` passed (then throws). */
+export async function waitFor(what: string, condition: () => Promise<boolean>, timeoutMs = 30_000) {
+  const until = Date.now() + timeoutMs
+  while (Date.now() < until) {
+    if (await condition()) return
+    await new Promise((r) => setTimeout(r, 100))
+  }
+  throw new Error(`Timed out waiting for ${what}`)
+}
+
+const BASE_MS = Date.UTC(2026, 9, 5, 10)
+
+/** A file name without digits (a number sequence would make an album of a poster's files): letters from `k`. */
+export function nameOf(k: number): string {
+  let s = ''
+  let n = k
+  do {
+    s = String.fromCharCode(97 + (n % 26)) + s
+    n = Math.floor(n / 26)
+  } while (n > 0)
+  return `Set.${s}.mkv`
+}
+
+/**
+ * The overview lines of articles `from`..`to`, one an hour from 2026-10-05: article 1 is a one-part file, and from 2
+ * on each even article is part 1 of a two-part file whose part 2 is the next (odd) article — so a file spans every
+ * chunk boundary at an even `to` (100 and 101, 200 and 201: the window has to carry it). Dates an hour apart make
+ * the posting frontier move a day every 24 articles, so the window closes on files as it goes.
+ */
+export function overviewLines(from: number, to: number): string[] {
+  const lines: string[] = []
+  for (let n = from; n <= to; n++) {
+    const date = new Date(BASE_MS + n * 3600_000).toUTCString().replace(' GMT', ' +0000')
+    const [name, part] =
+      n === 1 ? [nameOf(1), '(1/1)'] : n % 2 === 0 ? [nameOf(n), '(1/2)'] : [nameOf(n - 1), '(2/2)']
+    lines.push(
+      `${n}\t"${name}" yEnc ${part}\tposter <p@x>\t${date}\t<a${n}@e2e>\t\t700000\t5000\tXref: news.example ${GROUP}:${n}`
+    )
+  }
+  return lines
+}
+
+/** One XOVER chunk as the news server sends it (deflated). */
+export function overviewChunk(from: number, to: number): Buffer {
+  return deflateSync(Buffer.from(overviewLines(from, to).join('\r\n') + '\r\n'))
+}