Rebuild: a chunk's deadline doubles each time it's hit
Test and deploy / test (push) Successful in 25s
Test and deploy / deploy (push) Successful in 1m8s

At 90 s a chunk that always takes longer would be given up forever
(range 155437's worker gave up waiting on its stream). Each deadline a
chunk hits doubles its next one (90, 180, 360 ... up to 16x), reset
once it comes in.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-10 16:36:36 +02:00
1 parent d81a202172
commit dbccec932c
1 file changed
+8 -2
+8 -2
View File
@@ -59,6 +59,8 @@ export class Prefetcher {
private fetchingNow = 0
private writingNow = 0
private sampler?: NodeJS.Timeout
/** Deadlines a chunk hit, by range and chunk start: each doubles its next deadline, so a slow chunk still comes. */
private lateChunks = new Map<string, number>()
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
@@ -190,11 +192,14 @@ export class Prefetcher {
const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
this.inFlight.add(entry)
let late = false
const key = `${rangeId}:${span.from}`
const deadlineMs = coordinatorConfig.nntpChunkDeadlineMs * 2 ** Math.min(this.lateChunks.get(key) ?? 0, 4)
const deadline = setTimeout(() => {
late = true
this.stats.deadlines++
this.lateChunks.set(key, (this.lateChunks.get(key) ?? 0) + 1)
c.close()
}, coordinatorConfig.nntpChunkDeadlineMs)
}, deadlineMs)
let data: Buffer
let groupMs = 0
try {
@@ -204,11 +209,12 @@ export class Prefetcher {
data = await c.xoverRaw(span.from, span.to)
} catch (error) {
if (!late) throw error
throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} not in after ${coordinatorConfig.nntpChunkDeadlineMs / 1000} s; fetching it again`)
throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} not in after ${deadlineMs / 1000} s; fetching it again`)
} finally {
clearTimeout(deadline)
this.inFlight.delete(entry)
}
this.lateChunks.delete(key)
if (!cancelled) {
const t = c.lastTiming
this.stats.requests++