Rebuild: coordinator:run --shard=i/n splits prefetching over processes
The coordinator's event loop was 16.5% busy at ~10 MB/s of compressed XOVER (socket reads, buffers, GC, spool writes; nothing stands out), so one process tops out around 60 MB/s: less than 50 connections bring (~1M articles/s, ~70-86 MB/s). A shard prefetches the ranges with id % n = i - 1 with its own NNTP_CONNECTIONS; only shard 1 plans, merges and settles. Two shards of 5 connections fetched what one of 10 did. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
462da5c225
commit
69f1cc63b2
3 files changed
+30
-11
No files matched your search
@@ -28,6 +28,13 @@ const SIZE_MS = 30_000
|
||||
* kept its worker waiting (LOG.md 2026-10-09).
|
||||
*/
|
||||
export class Prefetcher {
|
||||
/**
|
||||
* This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
|
||||
* process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
|
||||
* LOG.md 2026-10-09), less than 50 connections bring.
|
||||
*/
|
||||
constructor(private shard = { index: 0, count: 1 }) {}
|
||||
|
||||
private targets: Target[] = []
|
||||
private claimed = new Set<string>()
|
||||
private running = 0
|
||||
@@ -126,13 +133,14 @@ export class Prefetcher {
|
||||
this.sizedAt = Date.now()
|
||||
this.spoolBytes = await Spool.size()
|
||||
}
|
||||
const rows = (q: any) => q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id')
|
||||
const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
|
||||
const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id'))
|
||||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||||
const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
|
||||
const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
const [{ ready }] = await db.from('ranges').where('status', 'planned').where('spooled_chunks', '>', 0).count('* as ready')
|
||||
const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
|
||||
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
|
||||
const fresh = room && Number(ready) < ahead
|
||||
? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead - Number(ready))
|
||||
|
||||
Reference in new issue
Block a user