The par2 prune's cursor is (created_at to the microsecond, set_id): 20k sets of one flush share a created_at
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f2c5607ec3
commit
a5e0399ee6
3 files changed
+54
-8
No files matched your search
@@ -26,6 +26,9 @@ export interface PruneRound {
|
||||
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
|
||||
*/
|
||||
export class Par2Pruner {
|
||||
/** Sets per batch (BATCH; smaller in the tests). */
|
||||
constructor(private batch = BATCH) {}
|
||||
|
||||
/**
|
||||
* Batches until done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from it): run
|
||||
* from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md 2026-10-17 (72)).
|
||||
@@ -33,14 +36,18 @@ export class Par2Pruner {
|
||||
async round(budgetMs = Infinity): Promise<PruneRound> {
|
||||
const deadline = Date.now() + budgetMs
|
||||
const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
|
||||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||
let since: Date = saved ? new Date(saved.at) : new Date(0)
|
||||
// (created_at to the microsecond, set_id): one prober flush gives thousands of sets the same created_at, and a
|
||||
// timestamp read back to the millisecond took the same batch again every round (LOG.md 2026-10-17 (73)).
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT at::text AS at, key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let since: string = cursors[0]?.at ?? '1970-01-01 00:00:00+00'
|
||||
let sinceKey: string = cursors[0]?.key ?? ''
|
||||
const until = new Date(Date.now() - SETTLED_MS)
|
||||
for (;;) {
|
||||
const { rows: sets } = await db.rawQuery(
|
||||
`SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
|
||||
HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
|
||||
[since, until, BATCH]
|
||||
`SELECT set_id, max(created_at)::text AS at FROM par2_sets GROUP BY set_id
|
||||
HAVING (max(created_at), set_id::text) > (?::timestamptz, ?::text) AND max(created_at) <= ?
|
||||
ORDER BY max(created_at), set_id LIMIT ?`,
|
||||
[since, sinceKey, until, this.batch]
|
||||
)
|
||||
if (sets.length === 0) break
|
||||
const ids = sets.map((s: { set_id: string }) => s.set_id)
|
||||
@@ -56,11 +63,12 @@ export class Par2Pruner {
|
||||
round.complete += done.length
|
||||
round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
|
||||
since = sets.at(-1).at
|
||||
sinceKey = sets.at(-1).set_id
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, since]
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, ?::timestamptz, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, since, sinceKey]
|
||||
)
|
||||
if (sets.length < BATCH || Date.now() > deadline) break
|
||||
if (sets.length < this.batch || Date.now() > deadline) break
|
||||
}
|
||||
if (Date.now() > deadline) return round
|
||||
// Judged: no cursor (verdicts come and go); only sets with rows left.
|
||||
|
||||
Reference in new issue
Block a user