LEASE_CAPTURE_ONLY: group leases of the corpus windows' ranges only
Test and deploy / test (push) Successful in 1m6s
Test and deploy / edge (push) Successful in 2m44s
Test and deploy / deploy (push) Skipped

The lease bounds alone don't select a window: the history planner's oldest
buckets are priority 0-2 as well, so the window worker at 0..0 also took 2026
history at 0 (25k ranges in 40 groups, fetched live) while 1,739 ranges of the
2016 corpus window sat at voyager 2's leftover priorities (done when the window
was marked, planned again by the reset). With LEASE_CAPTURE_ONLY the lease, next
and the prefetcher's run-ahead take capture ranges only; the run-ahead now also
keeps to the lease bounds. Test: a group with history at 0 only isn't leased,
the history range after the window isn't handed out.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-26 23:26:31 +01:00
1 parent 29e0cc8f6e
commit 4af5e06400
6 files changed
+106 -19

No files matched your search

+16 -5
View File
@@ -430,7 +430,11 @@ export class Prefetcher {
await db
.from('group_leases')
.whereNotNull('node_id')
.where('heartbeat_at', '>=', new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000))
.where(
'heartbeat_at',
'>=',
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
)
.select('group_id')
).map((l) => Number(l.group_id))
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
@@ -456,8 +460,10 @@ export class Prefetcher {
const want = Math.max(ahead - Number(ready), short)
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
const full = await CatalogSpace.full()
// Ahead of each held group's cursor: its next ranges in article order, a range of every held group before any
// group's second; without held groups (no window worker running) nothing is fetched ahead for workers.
// Ahead of each held group's cursor: its next ranges in article order (within what the group leases hand out:
// LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY), a range of every held group before any group's second; without
// held groups (no window worker running) nothing is fetched ahead for workers.
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
const fresh =
room && want > 0 && held.length > 0
? await rows(db.query())
@@ -465,8 +471,12 @@ export class Prefetcher {
.where('r.spooled_chunks', 0)
.whereRaw('r.next_id <= r.end_id')
.whereIn('r.group_id', held)
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
.where((q: any) => (leaseCaptureOnly ? q.where('r.capture', true) : q))
.where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
.orderByRaw('row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id')
.orderByRaw(
'row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id'
)
.limit(want)
: []
const targets: Target[] = []
@@ -478,7 +488,8 @@ export class Prefetcher {
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span))) continue
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
continue
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
missing.push(span)
}