The discard sweep beside the lanes, each dropped signature swept once
Test and deploy / test (push) Successful in 2m57s
Test and deploy / deploy (push) Successful in 38m14s

After aa40222 the prober still idled: a dropped signature's range of the
ladder index is minutes of HDD (309 dropped, 242 with a poster tested on
the rows) and the sweep read the same ranges again every five minutes
inside the claim. Now each dropped class and signature is swept once
(swept_at, migration 7100; null again when the record drops it anew),
20 of each per sweep in rounds of 10k rows, in a loop of its own in the
prober beside the lanes and the namer (LOG.md (133)).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-11-02 21:12:51 +01:00
1 parent aa4022295a
commit 5d3b578322
5 files changed
+102 -28

No files matched your search

+55 -18
View File
@@ -315,7 +315,8 @@ class PendingLadder {
[JSON.stringify([...counts.values()])]
)
await trx.rawQuery(
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float),
swept_at = CASE WHEN s.dropped THEN s.swept_at END
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
[
@@ -362,7 +363,8 @@ class PendingLadder {
[JSON.stringify([...classes.values()])]
)
await trx.rawQuery(
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float),
swept_at = CASE WHEN c.dropped THEN c.swept_at END
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
[
@@ -377,26 +379,61 @@ class PendingLadder {
/**
* Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
* many. Driven from the dropped classes and signatures into the claim's index and the ladder's: a scan of the
* waiting rows testing each (44M on 2026-11-02) ran 9+ minutes on the HDD every sweep (LOG.md (133)).
* many. Each dropped class or signature is swept once (swept_at; null again when the record drops it anew),
* `perSweep` of them at most, each by its index (the claim's for a class, the ladder's for a signature, whose
* poster is tested on the rows) in rounds of `limit`. A sweep of a signature's rows takes minutes on the HDD,
* so the prober runs this beside the lanes, not in the claim (LOG.md (133)); the edge drops new releases of
* dropped signatures and classes itself, so one sweep is all a drop needs.
*/
async discard(limit = 10_000): Promise<number> {
async discard(limit = 10_000, perSweep = 20): Promise<number> {
const { samplePercent } = await RuleSets.current()
const { rows: byClass } = await db.rawQuery(
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_classes c
JOIN pending_releases p ON p.status = ? AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass
WHERE c.dropped LIMIT ?`,
[PENDING.waiting, limit]
let discarded = 0
const { rows: classes } = await db.rawQuery(
`SELECT tool, group_id, klass FROM pending_classes WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, klass LIMIT ?`,
[perSweep]
)
const { rows: bySignature } = await db.rawQuery(
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_signatures s
JOIN pending_releases p ON p.status = ? AND p.group_id = s.group_id AND p.tool = s.tool AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))
WHERE s.dropped LIMIT ?`,
[PENDING.waiting, limit]
for (const c of classes) {
let rows: any[]
do {
;({ rows } = await db.rawQuery(
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? LIMIT ?`,
[PENDING.waiting, c.tool, c.group_id, c.klass, limit]
))
discarded += await this.discardKeys(rows, samplePercent)
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
await db.rawQuery(
`UPDATE pending_classes SET swept_at = now() WHERE tool = ? AND group_id = ? AND klass = ?`,
[c.tool, c.group_id, c.klass]
)
}
const { rows: signatures } = await db.rawQuery(
`SELECT tool, group_id, poster FROM pending_signatures WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, poster LIMIT ?`,
[perSweep]
)
const keys = [
...new Set([...byClass, ...bySignature].map((r: any) => r.release_key as string)),
].filter((key: string) => !explored(key, samplePercent))
for (const s of signatures) {
let rows: any[]
do {
;({ rows } = await db.rawQuery(
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
WHERE status = ? AND group_id = ? AND tool = ? AND (?::text = '' OR coalesce(poster, '') = ?::text) LIMIT ?`,
[PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit]
))
discarded += await this.discardKeys(rows, samplePercent)
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
await db.rawQuery(
`UPDATE pending_signatures SET swept_at = now() WHERE tool = ? AND group_id = ? AND poster = ?`,
[s.tool, s.group_id, s.poster]
)
}
return discarded
}
/** The rows discarded (status 2), the explored sample excepted, their classes' waiting counts lowered; how many. */
private async discardKeys(rows: any[], samplePercent: number): Promise<number> {
const keys = rows
.map((r: any) => r.release_key as string)
.filter((key: string) => !explored(key, samplePercent))
if (keys.length === 0) return 0
const { rows: gone } = await db.rawQuery(
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)