releases:merge-names: keyset paging on (created_at, set_id)
Test and deploy / test (push) Successful in 50s
Test and deploy / edge (push) Successful in 1m45s
Test and deploy / deploy (push) Successful in 3m5s

Sets stored in one batch share created_at, so paging on created_at < last
skipped ~10k of 61k sets; the timestamp now also stays text so microseconds
survive the round trip.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-24 11:15:32 +02:00
1 parent a66b0980f4
commit 2dd5b4cd81
1 file changed
+9 -5
+9 -5
View File
@@ -20,16 +20,20 @@ export default class ReleasesMergeNames extends BaseCommand {
async run() {
const merger = new Par2Merger()
let before = '9999-12-31'
// Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text
// so microseconds survive the round trip.
let before = ['9999-12-31', '']
let looked = 0
let merges = 0
let releases = 0
const examples: string[] = []
while (this.sets === undefined || looked < this.sets) {
const { rows } = await db.rawQuery(
`SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE s.created_at < ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
GROUP BY s.set_id ORDER BY at DESC LIMIT ${this.dryRun ? 50 : 500}`,
[before]
`SELECT set_id, at::text AS at FROM (
SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
GROUP BY s.set_id) x
WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`,
before
)
if (rows.length === 0) break
for (const { set_id: setId } of rows) {
@@ -52,7 +56,7 @@ export default class ReleasesMergeNames extends BaseCommand {
}
}
}
before = rows.at(-1).at
before = [rows.at(-1).at, rows.at(-1).set_id]
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
}
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)