The discard sweep beside the lanes, each dropped signature swept once
Test and deploy / test (push) Successful in 2m57s
Test and deploy / deploy (push) Successful in 38m14s

After aa40222 the prober still idled: a dropped signature's range of the
ladder index is minutes of HDD (309 dropped, 242 with a poster tested on
the rows) and the sweep read the same ranges again every five minutes
inside the claim. Now each dropped class and signature is swept once
(swept_at, migration 7100; null again when the record drops it anew),
20 of each per sweep in rounds of 10k rows, in a loop of its own in the
prober beside the lanes and the namer (LOG.md (133)).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-11-02 21:12:51 +01:00
1 parent aa4022295a
commit 5d3b578322
5 files changed
+102 -28

No files matched your search

+12 -5
View File
@@ -97,7 +97,6 @@ export class Prober {
}
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
private namer = new PendingNamer()
private sweptAt = 0
async run() {
// A previous run's claims: one prober at a time.
@@ -106,6 +105,17 @@ export class Prober {
const released = await PendingLadder.release()
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
// The ladder's discard sweep beside the lanes: a dropped signature's rows take minutes on the HDD (LOG.md (133)).
const sweeper = (async () => {
while (!this.stopped) {
try {
this.stats.discarded += await PendingLadder.discard()
} catch (error: any) {
logger.error(`Probing: the discard sweep failed: ${error.message}`)
}
for (let waited = 0; waited < SWEEP_MS && !this.stopped; waited += 1000) await sleep(1000)
}
})()
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
const flusher = (async () => {
while (!this.stopped || this.outcomes.length > 0) {
@@ -136,6 +146,7 @@ export class Prober {
}
await Promise.all(this.lanes)
await namer
await sweeper
await this.share.leave().catch(() => {})
// What was claimed and not probed goes back.
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
@@ -266,10 +277,6 @@ export class Prober {
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
private async claimPending() {
if (Date.now() - this.sweptAt > SWEEP_MS) {
this.sweptAt = Date.now()
this.stats.discarded += await PendingLadder.discard()
}
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
const probes = await PendingLadder.claim(want)
if (probes.length === 0) return