Prober: claim from random points of the queue, not in id order
In id order the prober went group by group, and some groups' articles take 0.6-3 s each (a.b.zogg 2016; repeated fetches the same, pipelining 4 deep no faster), where a sample across all years takes 20-60 ms for most: 7 articles/s on 6 connections, against 67/s in its first minutes. A claim now takes 20 slices from random points (353 ms per 200). tools/body_timing.ts: BODY time for the oldest and newest candidates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4ed7f89951
commit
18f5d27413
2 files changed
+53
-5
No files matched your search
+16
-5
@@ -19,6 +19,8 @@ type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'w
|
||||
/** Kinds in the order they're probed: a random name's real name first, then par2 sets, then archives. */
|
||||
const KINDS = ['lone', 'par2', 'archive']
|
||||
const FLUSH_MS = 2000
|
||||
/** Slices a claim takes from random points of the queue. */
|
||||
const SLICES = 20
|
||||
const REPORT_MS = 60_000
|
||||
|
||||
/**
|
||||
@@ -114,18 +116,27 @@ export class Prober {
|
||||
return this.queue.shift()
|
||||
}
|
||||
|
||||
/** Waiting candidates of settled releases, by kind, claimed so another round doesn't take them. */
|
||||
/**
|
||||
* Waiting candidates of settled releases, by kind, claimed so another round
|
||||
* doesn't take them: SLICES slices from random points of the queue. In id
|
||||
* order it went group by group, and some groups' articles take 0.6–3 s
|
||||
* where most take 20–60 ms (a.b.zogg 2016: 7 articles/s on 6 connections).
|
||||
*/
|
||||
private async claim() {
|
||||
for (const kind of KINDS) {
|
||||
const want = coordinatorConfig.probe.batch - this.queue.length
|
||||
if (want <= 0) return
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE probe_candidates SET status = 'taken' WHERE id IN (
|
||||
SELECT pc.id FROM probe_candidates pc JOIN releases r ON r.id = pc.release_id
|
||||
WHERE pc.status = 'waiting' AND pc.kind = ? AND r.settled_at IS NOT NULL
|
||||
ORDER BY pc.id LIMIT ? FOR UPDATE OF pc SKIP LOCKED)
|
||||
SELECT pc.id FROM (SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates) b
|
||||
CROSS JOIN LATERAL generate_series(1, ?) AS s
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT pc.id FROM probe_candidates pc JOIN releases r ON r.id = pc.release_id
|
||||
WHERE pc.status = 'waiting' AND pc.kind = ? AND r.settled_at IS NOT NULL
|
||||
AND pc.id >= b.lo + floor(random() * (b.hi - b.lo + 1))::bigint + s * 0
|
||||
ORDER BY pc.id LIMIT ? FOR UPDATE OF pc SKIP LOCKED) pc)
|
||||
RETURNING id, release_id, kind, message_id`,
|
||||
[kind, want]
|
||||
[SLICES, kind, Math.ceil(want / SLICES)]
|
||||
)
|
||||
for (const r of rows) this.queue.push({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id })
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user