Rebuild: coordinator:run --shard=i/n splits prefetching over processes

The coordinator's event loop was 16.5% busy at ~10 MB/s of compressed
XOVER (socket reads, buffers, GC, spool writes; nothing stands out), so
one process tops out around 60 MB/s: less than 50 connections bring
(~1M articles/s, ~70-86 MB/s). A shard prefetches the ranges with
id % n = i - 1 with its own NNTP_CONNECTIONS; only shard 1 plans, merges
and settles. Two shards of 5 connections fetched what one of 10 did.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 03:36:24 +02:00
1 parent 462da5c225
commit 69f1cc63b2
3 files changed
+30 -11

No files matched your search

+10 -2
View File
@@ -28,6 +28,13 @@ const SIZE_MS = 30_000
* kept its worker waiting (LOG.md 2026-10-09).
*/
export class Prefetcher {
/**
* This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
* process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
* LOG.md 2026-10-09), less than 50 connections bring.
*/
constructor(private shard = { index: 0, count: 1 }) {}
private targets: Target[] = []
private claimed = new Set<string>()
private running = 0
@@ -126,13 +133,14 @@ export class Prefetcher {
this.sizedAt = Date.now()
this.spoolBytes = await Spool.size()
}
const rows = (q: any) => q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id')
const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id'))
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
const [{ ready }] = await db.from('ranges').where('status', 'planned').where('spooled_chunks', '>', 0).count('* as ready')
const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
const fresh = room && Number(ready) < ahead
? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead - Number(ready))