The discard sweep beside the lanes, each dropped signature swept once
After aa40222 the prober still idled: a dropped signature's range of the
ladder index is minutes of HDD (309 dropped, 242 with a poster tested on
the rows) and the sweep read the same ranges again every five minutes
inside the claim. Now each dropped class and signature is swept once
(swept_at, migration 7100; null again when the record drops it anew),
20 of each per sweep in rounds of 10k rows, in a loop of its own in the
prober beside the lanes and the namer (LOG.md (133)).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
aa4022295a
commit
5d3b578322
5 files changed
+102
-28
No files matched your search
+12
-5
@@ -97,7 +97,6 @@ export class Prober {
|
||||
}
|
||||
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
|
||||
private namer = new PendingNamer()
|
||||
private sweptAt = 0
|
||||
|
||||
async run() {
|
||||
// A previous run's claims: one prober at a time.
|
||||
@@ -106,6 +105,17 @@ export class Prober {
|
||||
const released = await PendingLadder.release()
|
||||
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
|
||||
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
|
||||
// The ladder's discard sweep beside the lanes: a dropped signature's rows take minutes on the HDD (LOG.md (133)).
|
||||
const sweeper = (async () => {
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
} catch (error: any) {
|
||||
logger.error(`Probing: the discard sweep failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < SWEEP_MS && !this.stopped; waited += 1000) await sleep(1000)
|
||||
}
|
||||
})()
|
||||
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
|
||||
const flusher = (async () => {
|
||||
while (!this.stopped || this.outcomes.length > 0) {
|
||||
@@ -136,6 +146,7 @@ export class Prober {
|
||||
}
|
||||
await Promise.all(this.lanes)
|
||||
await namer
|
||||
await sweeper
|
||||
await this.share.leave().catch(() => {})
|
||||
// What was claimed and not probed goes back.
|
||||
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
@@ -266,10 +277,6 @@ export class Prober {
|
||||
|
||||
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
|
||||
private async claimPending() {
|
||||
if (Date.now() - this.sweptAt > SWEEP_MS) {
|
||||
this.sweptAt = Date.now()
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
}
|
||||
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
|
||||
const probes = await PendingLadder.claim(want)
|
||||
if (probes.length === 0) return
|
||||
|
||||
Reference in new issue
Block a user