Corpus prefetch: at most perGroup/perRange capture ranges per group
The prefetcher fetched capture ranges into the corpus 16 at a time in id order, and ids run group by group: the 16 were one or two groups', the per-group connection limit (4) let 4-8 loops work and left the rest idle, ~30k articles/s for the first batch of 19,955 capture ranges. Now at most perGroup / perRange (2) of a group are taken, so the 16 come from 8+ groups and every loop gets work (and the pool asks for more). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fb6b00f230
commit
c1946792b2
1 file changed
+10
-1
@@ -415,16 +415,24 @@ export class Prefetcher {
|
||||
return targets
|
||||
}
|
||||
|
||||
/** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
|
||||
/**
|
||||
* Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus; as many of a
|
||||
* group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which took 4 of the
|
||||
* loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)).
|
||||
*/
|
||||
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
|
||||
const { dir, ahead } = coordinatorConfig.corpus
|
||||
if (!dir || ahead <= 0) return []
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
const ofGroup = perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
|
||||
const groups = new Map<string, number>()
|
||||
const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
|
||||
const targets: Target[] = []
|
||||
for (const row of candidates) {
|
||||
if (targets.length >= ahead) break
|
||||
const id = Number(row.id)
|
||||
if (taken.has(id) || this.stored.has(id)) continue
|
||||
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
|
||||
const start = Number(row.start_id)
|
||||
const end = Number(row.end_id)
|
||||
const range = { group: row.group, start, end }
|
||||
@@ -437,6 +445,7 @@ export class Prefetcher {
|
||||
for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
|
||||
if (missing.length > 0) {
|
||||
targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
|
||||
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
|
||||
}
|
||||
}
|
||||
return targets
|
||||
|
||||
Reference in new issue
Block a user