Rebuild: a lease never waits for prefetching; one-shot hashes

A lease was only granted for a range with spooled chunks, and the
prefetcher kept ranges ahead per active lease: once the spool ran dry a
worker got none and slept 30 s, fewer ranges were leased, fewer
prefetched. Three workers held one lease between them (~50k headers/s).
A lease now takes a spooled range first, else the next planned one
(leased ranges are fetched first; the stream waits).

crypto.hash instead of createHash().update().digest() for file and
release keys: same digests, ~2.5x faster (~4.6% of a worker's time).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 03:08:19 +02:00
1 parent 168605148f
commit 9dc0a27bb4
5 files changed
+30 -25

No files matched your search

+20 -13
View File
@@ -7,23 +7,30 @@ import type { Lease } from '#lib/edge/index'
export class LeaseLost extends Error {}
/**
* Ranges for workers (REBUILD.md, coordinator): the next range whose first
* chunk is spooled, by priority (oldest era first); a lease whose worker
* went quiet for LEASE_TTL_SECONDS is taken over and resumes at its resume
* point.
* Ranges for workers (REBUILD.md, coordinator): the next range with chunks
* spooled, by priority (oldest era first), else the next planned one (the
* prefetcher fetches leased ranges first; the worker's stream waits). A
* worker never waits for prefetching to choose its range: refused leases
* left workers idle for 30 s, so fewer ranges were leased and fewer
* prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
* LEASE_TTL_SECONDS is taken over and resumes at its resume point.
*/
class Leases {
async lease(nodeId: number): Promise<Lease | null> {
const rules = await RuleSets.current()
const { rows } = await db.rawQuery(
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
WHERE r.id = (
SELECT id FROM ranges
WHERE spooled_chunks > 0 AND next_id <= end_id AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds]
)
const take = (spooled: boolean) =>
db.rawQuery(
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
WHERE r.id = (
SELECT id FROM ranges
WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id
AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds]
)
let { rows } = await take(true)
if (rows.length === 0) ({ rows } = await take(false))
const row = rows[0]
if (!row) return null
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })