The window worker: group leases, the stream serving the corpus, the snapshot through the API (design 1 and 5)

A worker thread leases a group and takes its ranges in article order through
one carried window; the coordinator keeps one cursor a group (group_leases,
migration 6500) and hands out the next range, the rules and nothing about
where the chunks are: the range stream serves a stored range's chunks from the
corpus file and a live one's from the spool. The window's snapshot (bincode:
the frontier, the open files, releases and albums, the upload numbers, the
batch-counter table) goes to the coordinator every EDGE_SNAPSHOT_MINUTES and
when the group is given back, and the next holder restores it; a takeover
after LEASE_TTL_SECONDS plans the ranges from the snapshot's article again.
The prefetcher runs ahead of held groups' cursors and skips chunks the corpus
has. LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY bound what is leased. edge-node
holds groups by default; --ranges keeps the old per-range path.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/group_leases.spec.ts

@@ -0,0 +1,145 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+import GroupLeases from '#services/group_leases'
+import { LeaseLost } from '#services/leases'
+import { Readable } from 'node:stream'
+
+async function node(name: string) {
+  return Number((await db.table('nodes').insert({ name }).returning('id'))[0].id)
+}
+
+async function group(name: string, priority: number) {
+  return Number((await db.table('groups').insert({ name, priority }).returning('id'))[0].id)
+}
+
+async function range(groupId: number, start: number, priority: number, extra: Record<string, unknown> = {}) {
+  const [row] = await db
+    .table('ranges')
+    .insert({ group_id: groupId, start_id: start, end_id: start + 99, next_id: start, priority, ...extra })
+    .returning('id')
+  return Number(row.id)
+}
+
+const rangeRow = async (id: number) => (await db.from('ranges').where('id', id).firstOrFail()) as any
+
+test.group('Group leases', (g) => {
+  let dir = ''
+  let snapshotDir = ''
+  g.each.setup(async () => {
+    const rollback = await testUtils.db().withGlobalTransaction()
+    dir = await mkdtemp(join(tmpdir(), 'snapshots-'))
+    snapshotDir = coordinatorConfig.snapshotDir
+    coordinatorConfig.snapshotDir = dir
+    return rollback
+  })
+  g.each.teardown(async () => {
+    coordinatorConfig.snapshotDir = snapshotDir
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  test('a lease takes the group with the most urgent planned range; its ranges come in article order; caught up it gives nothing', async ({
+    assert,
+  }) => {
+    const a = await group('alt.binaries.gl.a', 1)
+    const b = await group('alt.binaries.gl.b', 2)
+    // Group a: history at priority 0 and 1; group b: a tail range (priority -3) and history.
+    const a1 = await range(a, 1, 0)
+    const a2 = await range(a, 101, 0)
+    const a3 = await range(a, 201, 1)
+    const b2 = await range(b, 101, 0)
+    const b1 = await range(b, 1, -3)
+    const n1 = await node('gl-1')
+    const n2 = await node('gl-2')
+
+    const first = await GroupLeases.lease(n1)
+    assert.equal(first?.group.id, b, 'the tail first')
+    assert.isNull(first?.snapshot)
+    // Held: the next node gets the other group; a third gets nothing.
+    const second = await GroupLeases.lease(n2)
+    assert.equal(second?.group.id, a)
+    assert.isNull(await GroupLeases.lease(await node('gl-3')))
+
+    // Ranges in article order, whatever their priority.
+    assert.deepEqual((await GroupLeases.next(b, n1))?.range, { id: b1, start: 1, end: 100, next: 1 })
+    assert.include(await rangeRow(b1), { status: 'leased', node_id: n1 })
+    // Asked again before finishing: the same range.
+    assert.equal((await GroupLeases.next(b, n1))?.range.id, b1)
+    await db.from('ranges').where('id', b1).update({ status: 'done', next_id: 101 })
+    assert.equal((await GroupLeases.next(b, n1))?.range.id, b2)
+    await db.from('ranges').where('id', b2).update({ status: 'done', next_id: 201 })
+    assert.isNull(await GroupLeases.next(b, n1), 'caught up')
+
+    assert.deepEqual([a1, a2, a3].length, 3)
+    await assert.rejects(() => GroupLeases.next(a, n1), LeaseLost)
+  })
+
+  test('a stale holder is taken over and the ranges after the snapshot are read again; a release keeps the resume point', async ({
+    assert,
+  }) => {
+    const a = await group('alt.binaries.gl.c', 1)
+    const r1 = await range(a, 1, 0)
+    const r2 = await range(a, 101, 0)
+    const r3 = await range(a, 201, 0)
+    const r4 = await range(a, 301, 0)
+    const n1 = await node('gl-4')
+    const n2 = await node('gl-5')
+    assert.equal((await GroupLeases.lease(n1))?.group.id, a)
+    // The worker did r1 and r2, snapshotted at 201 (r3's start), then got halfway through r3 and went quiet.
+    await db.from('ranges').whereIn('id', [r1, r2]).update({ status: 'done', next_id: db.raw('end_id + 1') })
+    await GroupLeases.putSnapshot(a, n1, 201, Readable.from([Buffer.from('window at 201')]))
+    await GroupLeases.next(a, n1)
+    await db.from('ranges').where('id', r3).update({ next_id: 251 })
+    await db.from('group_leases').where('group_id', a).update({ heartbeat_at: new Date(Date.now() - 24 * 3600_000) })
+
+    const lease = await GroupLeases.lease(n2)
+    assert.equal(lease?.group.id, a)
+    assert.deepEqual(lease?.snapshot, { next: 201 })
+    // r1 and r2 stay done (before the snapshot); r3 is read again from the snapshot's article, r4 untouched.
+    assert.include(await rangeRow(r1), { status: 'done' })
+    assert.include(await rangeRow(r2), { status: 'done' })
+    assert.include(await rangeRow(r3), { status: 'planned', node_id: null })
+    assert.equal(Number((await rangeRow(r3)).next_id), 201)
+    assert.include(await rangeRow(r4), { status: 'planned' })
+    const snapshot = await GroupLeases.getSnapshot(a, n2)
+    assert.equal(snapshot?.next, 201)
+    assert.equal(snapshot?.size, 'window at 201'.length)
+    await assert.rejects(() => GroupLeases.next(a, n1), LeaseLost)
+
+    // The new holder works r3, snapshots at 301 and gives the group back: r3 done, r4 waits at its start.
+    assert.equal((await GroupLeases.next(a, n2))?.range.id, r3)
+    await db.from('ranges').where('id', r3).update({ status: 'done', next_id: 301 })
+    await GroupLeases.putSnapshot(a, n2, 301, Readable.from([Buffer.from('window at 301')]))
+    assert.equal((await GroupLeases.next(a, n2))?.range.id, r4)
+    await db.from('ranges').where('id', r4).update({ next_id: 301 })
+    await GroupLeases.release(a, n2)
+    assert.include(await rangeRow(r4), { status: 'planned', node_id: null })
+    assert.isNull((await db.from('group_leases').where('group_id', a).firstOrFail()).node_id)
+    // Leased again at once (free, not stale): the snapshot at 301 covers everything done.
+    const again = await GroupLeases.lease(n1)
+    assert.deepEqual(again?.snapshot, { next: 301 })
+    assert.include(await rangeRow(r3), { status: 'done' })
+    assert.equal((await GroupLeases.next(a, n1))?.range.id, r4)
+  })
+
+  test('a snapshot older than a done range makes that range be read again from its start', async ({ assert }) => {
+    const a = await group('alt.binaries.gl.d', 1)
+    const r1 = await range(a, 1, 0)
+    const r2 = await range(a, 101, 0)
+    const n1 = await node('gl-6')
+    await GroupLeases.lease(n1)
+    // The worker snapshotted at the start, finished r1, and crashed before its next snapshot: r1 is read again.
+    await GroupLeases.putSnapshot(a, n1, 1, Readable.from([Buffer.from('window at 1')]))
+    await db.from('ranges').where('id', r1).update({ status: 'done', next_id: 101 })
+    await db.from('group_leases').where('group_id', a).update({ heartbeat_at: new Date(Date.now() - 24 * 3600_000) })
+    await GroupLeases.lease(await node('gl-7'))
+    assert.equal((await rangeRow(r1)).status, 'planned')
+    assert.equal(Number((await rangeRow(r1)).next_id), 1)
+    assert.equal((await rangeRow(r2)).status, 'planned')
+    assert.equal(Number((await rangeRow(r2)).next_id), 101)
+  })
+})

tests/functional/node_api.spec.ts

@@ -7,6 +7,7 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import Node from '#models/node'
 import Spool from '#services/spool'
+import Corpus from '#services/corpus'
 import coordinatorConfig from '#config/coordinator'
 import { decodeFrames, fromWire, toWire, RangeIndexer, type Lease } from '#lib/edge/index'
 import { parseOverviews } from '#lib/nntp/index'
@@ -86,6 +87,72 @@ test.group('Node API', (group) => {
     assert.equal(range.status, 'done')
     assert.isFalse(await Spool.has(rangeId, { from: 1, to: 100 }))
   })
+  test('a group lease: the next range in order, its stream from the corpus, the snapshot through the API', async ({ client, assert }) => {
+    const corpus = await mkdtemp(join(tmpdir(), 'corpus-'))
+    const snapshots = await mkdtemp(join(tmpdir(), 'snapshots-'))
+    const saved = { corpus: coordinatorConfig.corpus.dir, snapshots: coordinatorConfig.snapshotDir }
+    coordinatorConfig.corpus.dir = corpus
+    coordinatorConfig.snapshotDir = snapshots
+    try {
+      const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
+      // Two ranges of the group: the first stored in the corpus (no spool), the second not.
+      const [r1] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 200, next_id: 1, capture: true, stored_at: new Date() }).returning('id')
+      const [r2] = await db.table('ranges').insert({ group_id: g.id, start_id: 201, end_id: 400, next_id: 201 }).returning('id')
+      const corpusRange = { group: GROUP, start: 1, end: 200 }
+      await Corpus.write(corpusRange, { from: 1, to: 100 }, chunk(11), 2)
+      await Corpus.write(corpusRange, { from: 101, to: 200 }, chunk(111), 2)
+      const node = await Node.create({ name: 'n1' })
+      const token = (await Node.accessTokens.create(node)).value!.release()
+
+      const lease = await client.post('/api/v1/node/groups/lease').bearerToken(token)
+      lease.assertStatus(200)
+      const groupId = Number(lease.body().group.id)
+      assert.equal(groupId, Number(g.id))
+      assert.isNull(lease.body().snapshot)
+      ;(await client.post('/api/v1/node/groups/lease').bearerToken(token)).assertStatus(204)
+
+      const next = await client.post(`/api/v1/node/groups/${groupId}/next`).bearerToken(token)
+      next.assertStatus(200)
+      assert.deepEqual(next.body().range, { id: Number(r1.id), start: 1, end: 200, next: 1 })
+
+      // The stored range's chunks come from the corpus, nothing spooled.
+      const stream = await client.get(`/api/v1/node/ranges/${r1.id}/stream`).bearerToken(token)
+      stream.assertStatus(200)
+      const raw = stream.body() as unknown
+      const body: Buffer = Buffer.isBuffer(raw) ? raw : Buffer.from(stream.text(), 'binary')
+      const frames = []
+      for await (const frame of decodeFrames([body])) frames.push(frame)
+      assert.deepEqual(frames.map((f) => [f.from, f.to]), [[1, 100], [101, 200]])
+      assert.equal(parseOverviews(unzipSync(frames[0].data).toString('utf8')).length, 3)
+
+      // The snapshot, standing at the second range's start.
+      const putRaw = (url: string, bytes: Buffer) => {
+        const request = client.put(url).bearerToken(token)
+        request.request.set('Content-Type', 'application/octet-stream').send(bytes)
+        return request
+      }
+      ;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot`, Buffer.from('window'))).assertStatus(400)
+      ;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot?next=201`, Buffer.from('window at 201'))).assertStatus(200)
+      await db.from('ranges').where('id', r1.id).update({ status: 'done', next_id: 201 })
+      const got = await client.get(`/api/v1/node/groups/${groupId}/snapshot`).bearerToken(token)
+      got.assertStatus(200)
+      assert.equal(got.header('x-snapshot-next'), '201')
+      const gotRaw = got.body() as unknown
+      assert.equal((Buffer.isBuffer(gotRaw) ? gotRaw : Buffer.from(got.text(), 'binary')).toString(), 'window at 201')
+      assert.equal(Number((await client.post(`/api/v1/node/groups/${groupId}/next`).bearerToken(token)).body().range.id), Number(r2.id))
+
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(200)
+      ;(await client.post(`/api/v1/node/groups/${groupId}/release`).bearerToken(token)).assertStatus(200)
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
+      assert.include(await db.from('ranges').where('id', r2.id).firstOrFail(), { status: 'planned', node_id: null })
+    } finally {
+      coordinatorConfig.corpus.dir = saved.corpus
+      coordinatorConfig.snapshotDir = saved.snapshots
+      await rm(corpus, { recursive: true, force: true })
+      await rm(snapshots, { recursive: true, force: true })
+    }
+  })
+
   test('a chunk that does not inflate: reported, removed from the spool, the range goes back', async ({ client, assert }) => {
     const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
     const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, spooled_chunks: 3 }).returning('id')