The discard sweep beside the lanes, each dropped signature swept once
After aa40222 the prober still idled: a dropped signature's range of the
ladder index is minutes of HDD (309 dropped, 242 with a poster tested on
the rows) and the sweep read the same ranges again every five minutes
inside the claim. Now each dropped class and signature is swept once
(swept_at, migration 7100; null again when the record drops it anew),
20 of each per sweep in rounds of 10k rows, in a loop of its own in the
prober beside the lanes and the namer (LOG.md (133)).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
aa4022295a
commit
5d3b578322
5 files changed
+102
-28
No files matched your search
@@ -315,7 +315,8 @@ class PendingLadder {
|
||||
[JSON.stringify([...counts.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float),
|
||||
swept_at = CASE WHEN s.dropped THEN s.swept_at END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||||
[
|
||||
@@ -362,7 +363,8 @@ class PendingLadder {
|
||||
[JSON.stringify([...classes.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
|
||||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float),
|
||||
swept_at = CASE WHEN c.dropped THEN c.swept_at END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[
|
||||
@@ -377,26 +379,61 @@ class PendingLadder {
|
||||
|
||||
/**
|
||||
* Waiting releases of a dropped class or signature discarded without a probe, the explored sample excepted; how
|
||||
* many. Driven from the dropped classes and signatures into the claim's index and the ladder's: a scan of the
|
||||
* waiting rows testing each (44M on 2026-11-02) ran 9+ minutes on the HDD every sweep (LOG.md (133)).
|
||||
* many. Each dropped class or signature is swept once (swept_at; null again when the record drops it anew),
|
||||
* `perSweep` of them at most, each by its index (the claim's for a class, the ladder's for a signature, whose
|
||||
* poster is tested on the rows) in rounds of `limit`. A sweep of a signature's rows takes minutes on the HDD,
|
||||
* so the prober runs this beside the lanes, not in the claim (LOG.md (133)); the edge drops new releases of
|
||||
* dropped signatures and classes itself, so one sweep is all a drop needs.
|
||||
*/
|
||||
async discard(limit = 10_000): Promise<number> {
|
||||
async discard(limit = 10_000, perSweep = 20): Promise<number> {
|
||||
const { samplePercent } = await RuleSets.current()
|
||||
const { rows: byClass } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_classes c
|
||||
JOIN pending_releases p ON p.status = ? AND p.tool = c.tool AND p.group_id = c.group_id AND p.klass = c.klass
|
||||
WHERE c.dropped LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
let discarded = 0
|
||||
const { rows: classes } = await db.rawQuery(
|
||||
`SELECT tool, group_id, klass FROM pending_classes WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, klass LIMIT ?`,
|
||||
[perSweep]
|
||||
)
|
||||
const { rows: bySignature } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_signatures s
|
||||
JOIN pending_releases p ON p.status = ? AND p.group_id = s.group_id AND p.tool = s.tool AND (s.poster = '' OR s.poster = coalesce(p.poster, ''))
|
||||
WHERE s.dropped LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
for (const c of classes) {
|
||||
let rows: any[]
|
||||
do {
|
||||
;({ rows } = await db.rawQuery(
|
||||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? LIMIT ?`,
|
||||
[PENDING.waiting, c.tool, c.group_id, c.klass, limit]
|
||||
))
|
||||
discarded += await this.discardKeys(rows, samplePercent)
|
||||
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET swept_at = now() WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||||
[c.tool, c.group_id, c.klass]
|
||||
)
|
||||
}
|
||||
const { rows: signatures } = await db.rawQuery(
|
||||
`SELECT tool, group_id, poster FROM pending_signatures WHERE dropped AND swept_at IS NULL ORDER BY group_id, tool, poster LIMIT ?`,
|
||||
[perSweep]
|
||||
)
|
||||
const keys = [
|
||||
...new Set([...byClass, ...bySignature].map((r: any) => r.release_key as string)),
|
||||
].filter((key: string) => !explored(key, samplePercent))
|
||||
for (const s of signatures) {
|
||||
let rows: any[]
|
||||
do {
|
||||
;({ rows } = await db.rawQuery(
|
||||
`SELECT encode(release_key, 'hex') AS release_key FROM pending_releases
|
||||
WHERE status = ? AND group_id = ? AND tool = ? AND (?::text = '' OR coalesce(poster, '') = ?::text) LIMIT ?`,
|
||||
[PENDING.waiting, s.group_id, s.tool, s.poster, s.poster, limit]
|
||||
))
|
||||
discarded += await this.discardKeys(rows, samplePercent)
|
||||
} while (rows.length === limit && rows.some((r) => !explored(r.release_key, samplePercent)))
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_signatures SET swept_at = now() WHERE tool = ? AND group_id = ? AND poster = ?`,
|
||||
[s.tool, s.group_id, s.poster]
|
||||
)
|
||||
}
|
||||
return discarded
|
||||
}
|
||||
|
||||
/** The rows discarded (status 2), the explored sample excepted, their classes' waiting counts lowered; how many. */
|
||||
private async discardKeys(rows: any[], samplePercent: number): Promise<number> {
|
||||
const keys = rows
|
||||
.map((r: any) => r.release_key as string)
|
||||
.filter((key: string) => !explored(key, samplePercent))
|
||||
if (keys.length === 0) return 0
|
||||
const { rows: gone } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||||
|
||||
Reference in new issue
Block a user