The window worker: group leases, the stream serving the corpus, the snapshot through the API (design 1 and 5)

A worker thread leases a group and takes its ranges in article order through
one carried window; the coordinator keeps one cursor a group (group_leases,
migration 6500) and hands out the next range, the rules and nothing about
where the chunks are: the range stream serves a stored range's chunks from the
corpus file and a live one's from the spool. The window's snapshot (bincode:
the frontier, the open files, releases and albums, the upload numbers, the
batch-counter table) goes to the coordinator every EDGE_SNAPSHOT_MINUTES and
when the group is given back, and the next holder restores it; a takeover
after LEASE_TTL_SECONDS plans the ranges from the snapshot's article again.
The prefetcher runs ahead of held groups' cursors and skips chunks the corpus
has. LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY bound what is leased. edge-node
holds groups by default; --ranges keeps the old per-range path.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/node_api.spec.ts

@@ -7,6 +7,7 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import Node from '#models/node'
 import Spool from '#services/spool'
+import Corpus from '#services/corpus'
 import coordinatorConfig from '#config/coordinator'
 import { decodeFrames, fromWire, toWire, RangeIndexer, type Lease } from '#lib/edge/index'
 import { parseOverviews } from '#lib/nntp/index'
@@ -86,6 +87,72 @@ test.group('Node API', (group) => {
     assert.equal(range.status, 'done')
     assert.isFalse(await Spool.has(rangeId, { from: 1, to: 100 }))
   })
+  test('a group lease: the next range in order, its stream from the corpus, the snapshot through the API', async ({ client, assert }) => {
+    const corpus = await mkdtemp(join(tmpdir(), 'corpus-'))
+    const snapshots = await mkdtemp(join(tmpdir(), 'snapshots-'))
+    const saved = { corpus: coordinatorConfig.corpus.dir, snapshots: coordinatorConfig.snapshotDir }
+    coordinatorConfig.corpus.dir = corpus
+    coordinatorConfig.snapshotDir = snapshots
+    try {
+      const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
+      // Two ranges of the group: the first stored in the corpus (no spool), the second not.
+      const [r1] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 200, next_id: 1, capture: true, stored_at: new Date() }).returning('id')
+      const [r2] = await db.table('ranges').insert({ group_id: g.id, start_id: 201, end_id: 400, next_id: 201 }).returning('id')
+      const corpusRange = { group: GROUP, start: 1, end: 200 }
+      await Corpus.write(corpusRange, { from: 1, to: 100 }, chunk(11), 2)
+      await Corpus.write(corpusRange, { from: 101, to: 200 }, chunk(111), 2)
+      const node = await Node.create({ name: 'n1' })
+      const token = (await Node.accessTokens.create(node)).value!.release()
+
+      const lease = await client.post('/api/v1/node/groups/lease').bearerToken(token)
+      lease.assertStatus(200)
+      const groupId = Number(lease.body().group.id)
+      assert.equal(groupId, Number(g.id))
+      assert.isNull(lease.body().snapshot)
+      ;(await client.post('/api/v1/node/groups/lease').bearerToken(token)).assertStatus(204)
+
+      const next = await client.post(`/api/v1/node/groups/${groupId}/next`).bearerToken(token)
+      next.assertStatus(200)
+      assert.deepEqual(next.body().range, { id: Number(r1.id), start: 1, end: 200, next: 1 })
+
+      // The stored range's chunks come from the corpus, nothing spooled.
+      const stream = await client.get(`/api/v1/node/ranges/${r1.id}/stream`).bearerToken(token)
+      stream.assertStatus(200)
+      const raw = stream.body() as unknown
+      const body: Buffer = Buffer.isBuffer(raw) ? raw : Buffer.from(stream.text(), 'binary')
+      const frames = []
+      for await (const frame of decodeFrames([body])) frames.push(frame)
+      assert.deepEqual(frames.map((f) => [f.from, f.to]), [[1, 100], [101, 200]])
+      assert.equal(parseOverviews(unzipSync(frames[0].data).toString('utf8')).length, 3)
+
+      // The snapshot, standing at the second range's start.
+      const putRaw = (url: string, bytes: Buffer) => {
+        const request = client.put(url).bearerToken(token)
+        request.request.set('Content-Type', 'application/octet-stream').send(bytes)
+        return request
+      }
+      ;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot`, Buffer.from('window'))).assertStatus(400)
+      ;(await putRaw(`/api/v1/node/groups/${groupId}/snapshot?next=201`, Buffer.from('window at 201'))).assertStatus(200)
+      await db.from('ranges').where('id', r1.id).update({ status: 'done', next_id: 201 })
+      const got = await client.get(`/api/v1/node/groups/${groupId}/snapshot`).bearerToken(token)
+      got.assertStatus(200)
+      assert.equal(got.header('x-snapshot-next'), '201')
+      const gotRaw = got.body() as unknown
+      assert.equal((Buffer.isBuffer(gotRaw) ? gotRaw : Buffer.from(got.text(), 'binary')).toString(), 'window at 201')
+      assert.equal(Number((await client.post(`/api/v1/node/groups/${groupId}/next`).bearerToken(token)).body().range.id), Number(r2.id))
+
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(200)
+      ;(await client.post(`/api/v1/node/groups/${groupId}/release`).bearerToken(token)).assertStatus(200)
+      ;(await client.post(`/api/v1/node/groups/${groupId}/heartbeat`).bearerToken(token)).assertStatus(409)
+      assert.include(await db.from('ranges').where('id', r2.id).firstOrFail(), { status: 'planned', node_id: null })
+    } finally {
+      coordinatorConfig.corpus.dir = saved.corpus
+      coordinatorConfig.snapshotDir = saved.snapshots
+      await rm(corpus, { recursive: true, force: true })
+      await rm(snapshots, { recursive: true, force: true })
+    }
+  })
+
   test('a chunk that does not inflate: reported, removed from the spool, the range goes back', async ({ client, assert }) => {
     const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
     const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, spooled_chunks: 3 }).returning('id')