LEASE_CAPTURE_ONLY: group leases of the corpus windows' ranges only
The lease bounds alone don't select a window: the history planner's oldest buckets are priority 0-2 as well, so the window worker at 0..0 also took 2026 history at 0 (25k ranges in 40 groups, fetched live) while 1,739 ranges of the 2016 corpus window sat at voyager 2's leftover priorities (done when the window was marked, planned again by the reset). With LEASE_CAPTURE_ONLY the lease, next and the prefetcher's run-ahead take capture ranges only; the run-ahead now also keeps to the lease bounds. Test: a group with history at 0 only isn't leased, the history range after the window isn't handed out. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
29e0cc8f6e
commit
4af5e06400
6 files changed
+106
-19
No files matched your search
@@ -430,7 +430,11 @@ export class Prefetcher {
|
||||
await db
|
||||
.from('group_leases')
|
||||
.whereNotNull('node_id')
|
||||
.where('heartbeat_at', '>=', new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000))
|
||||
.where(
|
||||
'heartbeat_at',
|
||||
'>=',
|
||||
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
|
||||
)
|
||||
.select('group_id')
|
||||
).map((l) => Number(l.group_id))
|
||||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||||
@@ -456,8 +460,10 @@ export class Prefetcher {
|
||||
const want = Math.max(ahead - Number(ready), short)
|
||||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||||
const full = await CatalogSpace.full()
|
||||
// Ahead of each held group's cursor: its next ranges in article order, a range of every held group before any
|
||||
// group's second; without held groups (no window worker running) nothing is fetched ahead for workers.
|
||||
// Ahead of each held group's cursor: its next ranges in article order (within what the group leases hand out:
|
||||
// LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY), a range of every held group before any group's second; without
|
||||
// held groups (no window worker running) nothing is fetched ahead for workers.
|
||||
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
|
||||
const fresh =
|
||||
room && want > 0 && held.length > 0
|
||||
? await rows(db.query())
|
||||
@@ -465,8 +471,12 @@ export class Prefetcher {
|
||||
.where('r.spooled_chunks', 0)
|
||||
.whereRaw('r.next_id <= r.end_id')
|
||||
.whereIn('r.group_id', held)
|
||||
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
|
||||
.where((q: any) => (leaseCaptureOnly ? q.where('r.capture', true) : q))
|
||||
.where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
|
||||
.orderByRaw('row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id')
|
||||
.orderByRaw(
|
||||
'row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id'
|
||||
)
|
||||
.limit(want)
|
||||
: []
|
||||
const targets: Target[] = []
|
||||
@@ -478,7 +488,8 @@ export class Prefetcher {
|
||||
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
|
||||
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
|
||||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span))) continue
|
||||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
|
||||
continue
|
||||
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
|
||||
missing.push(span)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user