Rebuild: the merger stops when a pass over the file keys decides nothing
Waiting files counted as work, so the merge loop re-read them for its whole 30 s budget every minute: Postgres used ~a core even without workers. Now ~4% at rest, and storing a boneless range costs Postgres ~0.7–1.9 s of CPU (it was 33 s with the loop running). Chunk payloads are stored EXTERNAL (zstd already; no pglz attempt). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
73fc9aabdf
commit
821de668d0
3 files changed
+15
-4
No files matched your search
@@ -48,14 +48,20 @@ export default class CoordinatorRun extends BaseCommand {
|
||||
}
|
||||
if (Date.now() - mergedAt >= MERGE_EVERY_MS) {
|
||||
mergedAt = Date.now()
|
||||
// Rounds until nothing more can be decided, or MERGE_BUDGET_MS.
|
||||
// Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
|
||||
// (Going on while only waiting files were left kept Postgres busy all the time.)
|
||||
const totals = { completed: 0, incomplete: 0, waiting: 0 }
|
||||
let decidedThisPass = 0
|
||||
while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
|
||||
const merged = await FragmentMerger.run()
|
||||
totals.completed += merged.completed
|
||||
totals.incomplete += merged.incomplete
|
||||
totals.waiting = merged.waiting
|
||||
if (merged.files === 0) break
|
||||
decidedThisPass += merged.completed + merged.incomplete
|
||||
if (merged.wrapped) {
|
||||
if (decidedThisPass === 0) break
|
||||
decidedThisPass = 0
|
||||
}
|
||||
}
|
||||
if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user