Rebuild: merger rounds with one neighbour query, a cursor and a time budget; planning skips failing groups
The neighbour check was three queries per file (~2,000 files/minute); now one per round of 20k files, with a cursor so waiting files don't take every round's places, and rounds until nothing is left or 30 s. On the dev backlog (old-style fragments): ~18k files/minute in the 30 s budget; at full throughput the new fragment rule leaves ~1,200 files/minute. A group that fails GROUP (gone from the server) no longer stops planning the others; edge:bench's groups are inactive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b862201991
commit
31493a98be
4 files changed
+66
-22
No files matched your search
@@ -9,6 +9,7 @@ import NntpPool from '#services/nntp_pool'
|
||||
const TICK_MS = 2000
|
||||
const PLAN_EVERY_MS = 60 * 60 * 1000
|
||||
const MERGE_EVERY_MS = 60 * 1000
|
||||
const MERGE_BUDGET_MS = 30 * 1000
|
||||
|
||||
/**
|
||||
* The coordinator's background work (REBUILD.md, part 1): the prefetcher,
|
||||
@@ -44,8 +45,16 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
}
|
||||
if (Date.now() - mergedAt >= MERGE_EVERY_MS) {
|
||||
mergedAt = Date.now()
|
||||
const merged = await FragmentMerger.run()
|
||||
if (merged.completed || merged.incomplete) this.logger.info(`Fragments: ${merged.completed} files completed, ${merged.incomplete} incomplete, ${merged.waiting} waiting.`)
|
||||
// Rounds until nothing more can be decided, or MERGE_BUDGET_MS.
|
||||
const totals = { completed: 0, incomplete: 0, waiting: 0 }
|
||||
while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
|
||||
const merged = await FragmentMerger.run()
|
||||
totals.completed += merged.completed
|
||||
totals.incomplete += merged.incomplete
|
||||
totals.waiting = merged.waiting
|
||||
if (merged.files === 0) break
|
||||
}
|
||||
if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
|
||||
}
|
||||
await prefetcher.tick()
|
||||
} catch (error: any) {
|
||||
|
||||
@@ -101,7 +101,8 @@ export default class EdgeBench extends BaseCommand {
|
||||
let rangeId = 0
|
||||
let nodeId = 0
|
||||
if (this.store) {
|
||||
const [g] = await db.table('groups').insert({ name: `bench:${group}:${Date.now()}`, priority: 1_000_000 + (Date.now() % 1_000_000) }).returning('id')
|
||||
// Inactive: the coordinator neither plans nor leases it.
|
||||
const [g] = await db.table('groups').insert({ name: `bench:${group}:${Date.now()}`, priority: 1_000_000 + (Date.now() % 1_000_000), active: false }).returning('id')
|
||||
const [n] = await db.table('nodes').insert({ name: `bench-${Date.now()}` }).returning('id')
|
||||
nodeId = Number(n.id)
|
||||
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: end, next_id: start, status: 'leased', node_id: nodeId }).returning('id')
|
||||
|
||||
Reference in new issue
Block a user