The window worker: group leases, the stream serving the corpus, the snapshot through the API (design 1 and 5)

A worker thread leases a group and takes its ranges in article order through
one carried window; the coordinator keeps one cursor a group (group_leases,
migration 6500) and hands out the next range, the rules and nothing about
where the chunks are: the range stream serves a stored range's chunks from the
corpus file and a live one's from the spool. The window's snapshot (bincode:
the frontier, the open files, releases and albums, the upload numbers, the
batch-counter table) goes to the coordinator every EDGE_SNAPSHOT_MINUTES and
when the group is given back, and the next holder restores it; a takeover
after LEASE_TTL_SECONDS plans the ranges from the snapshot's article again.
The prefetcher runs ahead of held groups' cursors and skips chunks the corpus
has. LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY bound what is leased. edge-node
holds groups by default; --ranges keeps the old per-range path.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/controllers/node_controller.ts

@@ -1,9 +1,12 @@
 import type { HttpContext } from '@adonisjs/core/http'
 import { Readable } from 'node:stream'
 import { setTimeout as sleep } from 'node:timers/promises'
+import db from '@adonisjs/lucid/services/db'
 import Leases, { LeaseLost } from '#services/leases'
+import GroupLeases from '#services/group_leases'
 import BatchStore, { StaleBatch } from '#services/batch_store'
 import Spool from '#services/spool'
+import Corpus from '#services/corpus'
 import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
 
 /**
@@ -30,15 +33,82 @@ export default class NodeController {
     return lease
   }
 
-  /** The range's chunks from its resume point, waiting for those still being fetched. */
+  // Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
+  // ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
+
+  async leaseGroup(ctx: HttpContext) {
+    const lease = await GroupLeases.lease(this.node(ctx))
+    if (!lease) return ctx.response.noContent()
+    return lease
+  }
+
+  async nextRange(ctx: HttpContext) {
+    try {
+      const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx))
+      if (!next) return ctx.response.noContent()
+      return next
+    } catch (error) {
+      if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  async groupHeartbeat(ctx: HttpContext) {
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId))
+  }
+
+  async releaseGroup(ctx: HttpContext) {
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId))
+  }
+
+  /**
+   * PUT /groups/:id/snapshot?next=N: the window's serialized state standing at article N, streamed to disk as sent
+   * (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
+   */
+  async putSnapshot(ctx: HttpContext) {
+    const next = Number(ctx.request.qs().next)
+    if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
+    const raw = ctx.request.raw()
+    const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
+    return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body))
+  }
+
+  async getSnapshot(ctx: HttpContext) {
+    try {
+      const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx))
+      if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
+      ctx.response.header('Content-Type', 'application/octet-stream')
+      ctx.response.header('Content-Length', String(snapshot.size))
+      ctx.response.header('X-Snapshot-Next', String(snapshot.next))
+      return ctx.response.stream(snapshot.stream)
+    } catch (error) {
+      if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  /**
+   * The range's chunks from its resume point: a stored range's from the corpus (a worker needs no mount; REBUILD.md
+   * design 1), else from the spool, waiting for those still being fetched.
+   */
   async stream(ctx: HttpContext) {
     const nodeId = this.node(ctx)
     const rangeId = Number(ctx.params.id)
     const range = await this.leased(ctx, rangeId, nodeId)
     if (!range) return
     const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
+    const group = range.capture ? await db.from('groups').where('id', range.group_id).select('name').first() : null
+    const corpusRange = group ? { group: group.name as string, start: Number(range.start_id), end: Number(range.end_id) } : null
     async function* frames() {
       for (const span of spans) {
+        if (corpusRange && (await Corpus.has(corpusRange, span))) {
+          const data = await Corpus.read(corpusRange, span)
+          if (data) {
+            yield encodeFrame({ ...span, data })
+            await Leases.heartbeat(rangeId, nodeId)
+            continue
+          }
+        }
         const waitUntil = Date.now() + CHUNK_WAIT_MS
         let quietSince = Date.now()
         while (!(await Spool.has(rangeId, span))) {
@@ -75,7 +145,11 @@ export default class NodeController {
   }
 
   async finish(ctx: HttpContext) {
-    return this.lifecycle(ctx, (rangeId, nodeId) => Leases.finish(rangeId, nodeId))
+    return this.lifecycle(ctx, async (rangeId, nodeId) => {
+      const range = await Leases.leased(rangeId, nodeId)
+      await Leases.finish(rangeId, nodeId)
+      await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers))
+    })
   }
 
   /** POST /ranges/:id/corrupt { from }: a chunk that doesn't inflate; the range goes back. */

app/services/group_leases.ts

@@ -0,0 +1,164 @@
+import db from '@adonisjs/lucid/services/db'
+import { createReadStream, createWriteStream } from 'node:fs'
+import { mkdir, rename, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import coordinatorConfig from '#config/coordinator'
+import RuleSets from '#services/rule_sets'
+import CatalogSpace from '#services/catalog_space'
+import { LeaseLost } from '#services/leases'
+import type { GroupLease, NextRange } from '#lib/edge/index'
+
+/**
+ * Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
+ * next range each time it finished one, so a group's ranges are indexed in article order through one carried
+ * window. The coordinator keeps one row a group (group_leases): the holder, its heartbeat, the range it works and
+ * the snapshot its window stands at.
+ *
+ * - lease: the free group with the most urgent planned range (the ranges' priority: the tail before history, then
+ *   the windows), the longest unheld first; a holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group
+ *   resets what the window's snapshot doesn't cover: the ranges from the snapshot's article on are planned again
+ *   (the batch store's keys make the redo idempotent); without a snapshot the window is lost and the group
+ *   continues at its resume point (one seam).
+ * - next: the group's lowest planned range (or the one this node already has), leased to the node; none when the
+ *   group is caught up: the worker snapshots and gives the group back, and gets it again when the tail planned
+ *   more.
+ * - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR), given to the next holder.
+ */
+class GroupLeases {
+  get dir(): string {
+    return coordinatorConfig.snapshotDir
+  }
+
+  async lease(nodeId: number): Promise<GroupLease | null> {
+    const rules = await RuleSets.current()
+    const ttl = coordinatorConfig.leaseTtlSeconds
+    const { leasePriorityMin: min, leasePriorityMax: max } = coordinatorConfig
+    // The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
+    const history = (await CatalogSpace.full()) ? 'AND r.priority < 0' : ''
+    return db.transaction(async (trx) => {
+      // One lease at a time: the choice reads several tables.
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
+      const { rows } = await trx.rawQuery(
+        `SELECT g.id, g.name, g.priority, l.node_id AS holder, l.snapshot_next, todo.priority AS todo_priority
+         FROM groups g
+         JOIN LATERAL (SELECT min(r.priority) AS priority FROM ranges r WHERE r.group_id = g.id AND r.status = 'planned' AND r.next_id <= r.end_id AND r.priority BETWEEN ? AND ? ${history}) todo ON true
+         LEFT JOIN group_leases l ON l.group_id = g.id
+         WHERE g.active AND todo.priority IS NOT NULL
+           AND (l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?))
+         ORDER BY todo.priority, l.heartbeat_at NULLS FIRST, g.priority
+         LIMIT 1`,
+        [min, max, ttl]
+      )
+      const row = rows[0]
+      if (!row) return null
+      const groupId = Number(row.id)
+      // What the snapshot doesn't cover is read again: the ranges from its article on, the one holding it from there.
+      if (row.snapshot_next !== null) {
+        await trx.rawQuery(
+          `UPDATE ranges SET status = 'planned', node_id = NULL, next_id = greatest(start_id, ?::bigint)
+           WHERE group_id = ? AND end_id >= ?::bigint AND (status IN ('leased', 'done') OR next_id > ?::bigint)`,
+          [row.snapshot_next, groupId, row.snapshot_next, row.snapshot_next]
+        )
+      } else {
+        await trx.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL WHERE group_id = ? AND status = 'leased'`, [groupId])
+      }
+      await trx.rawQuery(
+        `INSERT INTO group_leases (group_id, node_id, leased_at, heartbeat_at, range_id)
+         VALUES (?, ?, now(), now(), NULL)
+         ON CONFLICT (group_id) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
+        [groupId, nodeId]
+      )
+      await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
+      return {
+        group: { id: groupId, name: row.name, priority: Number(row.priority) },
+        rules,
+        snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
+      }
+    })
+  }
+
+  /** The group's lease row, when this node holds it. */
+  async held(groupId: number, nodeId: number) {
+    const lease = await db.from('group_leases').where('group_id', groupId).first()
+    if (!lease || Number(lease.node_id) !== nodeId) throw new LeaseLost(`Group ${groupId} is not held by this node`)
+    return lease
+  }
+
+  /** The group's next range in article order (or the one this node has), leased to the node; null when caught up. */
+  async next(groupId: number, nodeId: number): Promise<NextRange | null> {
+    await this.held(groupId, nodeId)
+    const rules = await RuleSets.current()
+    const { leasePriorityMin: min, leasePriorityMax: max } = coordinatorConfig
+    const mine = await db.from('ranges').where('group_id', groupId).where('status', 'leased').where('node_id', nodeId).orderBy('start_id').first()
+    const row =
+      mine ??
+      (
+        await db.rawQuery(
+          `UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
+           WHERE r.id = (
+             SELECT id FROM ranges WHERE group_id = ? AND status = 'planned' AND next_id <= end_id AND priority BETWEEN ? AND ?
+             ORDER BY start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
+           RETURNING r.id, r.start_id, r.end_id, r.next_id`,
+          [nodeId, rules.version, groupId, min, max]
+        )
+      ).rows[0]
+    await db
+      .from('group_leases')
+      .where('group_id', groupId)
+      .update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
+    if (!row) return null
+    return { range: { id: Number(row.id), start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) } }
+  }
+
+  async heartbeat(groupId: number, nodeId: number) {
+    await this.held(groupId, nodeId)
+    await db.from('group_leases').where('group_id', groupId).update({ heartbeat_at: new Date() })
+    await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
+  }
+
+  /** A range of the group finished by its holder: counted on the lease (Leases.finish does the range). */
+  async rangeDone(groupId: number, headers: number) {
+    await db.rawQuery(`UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ?`, [headers, groupId])
+  }
+
+  /** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
+  async release(groupId: number, nodeId: number) {
+    await this.held(groupId, nodeId)
+    await db.from('ranges').where('group_id', groupId).where('status', 'leased').where('node_id', nodeId).update({ status: 'planned', node_id: null })
+    await db.from('group_leases').where('group_id', groupId).update({ node_id: null, range_id: null })
+  }
+
+  private snapshotPath(groupId: number) {
+    return join(this.dir, `${groupId}.gz`)
+  }
+
+  /**
+   * The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
+   * gigabytes) and written whole before it replaces the old one.
+   */
+  async putSnapshot(groupId: number, nodeId: number, next: number, body: Readable) {
+    await this.held(groupId, nodeId)
+    await mkdir(this.dir, { recursive: true })
+    const path = this.snapshotPath(groupId)
+    await pipeline(body, createWriteStream(`${path}.part`))
+    await rename(`${path}.part`, path)
+    await db.from('group_leases').where('group_id', groupId).update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
+  }
+
+  /** The snapshot's stream and size, or null when there is none. */
+  async getSnapshot(groupId: number, nodeId: number): Promise<{ stream: Readable; size: number; next: number } | null> {
+    const lease = await this.held(groupId, nodeId)
+    if (lease.snapshot_next === null) return null
+    const path = this.snapshotPath(groupId)
+    try {
+      const { size } = await stat(path)
+      return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }
+    } catch {
+      return null
+    }
+  }
+}
+
+export default new GroupLeases()

app/services/prefetcher.ts

@@ -425,10 +425,18 @@ export class Prefetcher {
       )
     // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
     const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
+    // The groups workers hold (group_leases; REBUILD.md design 1): the prefetcher runs ahead of their cursors.
+    const held: number[] = (
+      await db
+        .from('group_leases')
+        .whereNotNull('node_id')
+        .where('heartbeat_at', '>=', new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000))
+        .select('group_id')
+    ).map((l) => Number(l.group_id))
     // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
     const ahead = Math.max(
       coordinatorConfig.spool.minAhead,
-      coordinatorConfig.spool.aheadPerLease * leased.length
+      coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
     )
     const started = await rows(db.query())
       .where('r.status', 'planned')
@@ -448,14 +456,17 @@ export class Prefetcher {
     const want = Math.max(ahead - Number(ready), short)
     // The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
     const full = await CatalogSpace.full()
+    // Ahead of each held group's cursor: its next ranges in article order, a range of every held group before any
+    // group's second; without held groups (no window worker running) nothing is fetched ahead for workers.
     const fresh =
-      room && want > 0
+      room && want > 0 && held.length > 0
         ? await rows(db.query())
             .where('r.status', 'planned')
             .where('r.spooled_chunks', 0)
             .whereRaw('r.next_id <= r.end_id')
+            .whereIn('r.group_id', held)
             .where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
-            .orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
+            .orderByRaw('row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id')
             .limit(want)
         : []
     const targets: Target[] = []
@@ -464,9 +475,12 @@ export class Prefetcher {
       const end = Number(row.end_id)
       const total = Spool.chunksOf({ start, end }).length
       const missing: ChunkSpan[] = []
+      // A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
+      const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
       for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
-        if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span)))
-          missing.push(span)
+        if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span))) continue
+        if (corpusRange && (await Corpus.has(corpusRange, span))) continue
+        missing.push(span)
       }
       if (missing.length > 0) {
         targets.push({