From f4c0417066079226cd64f21397a946238f6ba58f Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sat, 10 Oct 2026 08:44:09 +0200 Subject: [PATCH] Rebuild: enough ranges being fetched for every connection With 2 connections per range the prefetcher needs 20+ ranges with chunks left; fully spooled ranges counted as ready (SPOOL_AHEAD_PER_LEASE) but take no connection, so on small ranges 30 of 40 connections were busy (13-15 ranges). Fresh ranges now also fill up to loops/perRange + 2 ranges being fetched. Co-Authored-By: Claude Opus 5.5 --- app/services/prefetcher.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/services/prefetcher.ts b/app/services/prefetcher.ts index 8048873..0b47ab2 100644 --- a/app/services/prefetcher.ts +++ b/app/services/prefetcher.ts @@ -341,8 +341,13 @@ export class Prefetcher { const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]) const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready') const room = this.spoolBytes < coordinatorConfig.spool.maxBytes - const fresh = room && Number(ready) < ahead - ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead - Number(ready)) + // With at most perRange connections a range, enough ranges still being fetched for all loops: fully + // spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)). + const { perRange } = coordinatorConfig.spool + const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 2 - (leased.length + started.length) : 0 + const want = Math.max(ahead - Number(ready), short) + const fresh = room && want > 0 + ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want) : [] const targets: Target[] = [] for (const row of [...leased, ...(room ? started : []), ...fresh]) {