From c1946792b22fe4c266ae77d2daeff6145baf01c0 Mon Sep 17 00:00:00 2001
From: Daan Meijer <daan@studioseptember.nl>
Date: Fri, 23 Oct 2026 15:25:18 +0200
Subject: [PATCH] Corpus prefetch: at most perGroup/perRange capture ranges per
 group

The prefetcher fetched capture ranges into the corpus 16 at a time in id
order, and ids run group by group: the 16 were one or two groups', the
per-group connection limit (4) let 4-8 loops work and left the rest
idle, ~30k articles/s for the first batch of 19,955 capture ranges. Now
at most perGroup / perRange (2) of a group are taken, so the 16 come
from 8+ groups and every loop gets work (and the pool asks for more).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 app/services/prefetcher.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/app/services/prefetcher.ts b/app/services/prefetcher.ts
index 846a042..7e3a26a 100644
--- a/app/services/prefetcher.ts
+++ b/app/services/prefetcher.ts
@@ -415,16 +415,24 @@ export class Prefetcher {
     return targets
   }
 
-  /** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
+  /**
+   * Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus; as many of a
+   * group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which took 4 of the
+   * loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)).
+   */
   private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
     const { dir, ahead } = coordinatorConfig.corpus
     if (!dir || ahead <= 0) return []
+    const { perRange, perGroup } = coordinatorConfig.spool
+    const ofGroup = perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
+    const groups = new Map<string, number>()
     const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
     const targets: Target[] = []
     for (const row of candidates) {
       if (targets.length >= ahead) break
       const id = Number(row.id)
       if (taken.has(id) || this.stored.has(id)) continue
+      if ((groups.get(row.group) ?? 0) >= ofGroup) continue
       const start = Number(row.start_id)
       const end = Number(row.end_id)
       const range = { group: row.group, start, end }
@@ -437,6 +445,7 @@ export class Prefetcher {
       for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
       if (missing.length > 0) {
         targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
+        groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
       }
     }
     return targets