From 2dd5b4cd8105299380f0d06e3f96668ec8f2264b Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sat, 24 Oct 2026 11:15:32 +0200 Subject: [PATCH] releases:merge-names: keyset paging on (created_at, set_id) Sets stored in one batch share created_at, so paging on created_at < last skipped ~10k of 61k sets; the timestamp now also stays text so microseconds survive the round trip. Co-Authored-By: Claude Opus 5.5 --- commands/releases_merge_names.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/commands/releases_merge_names.ts b/commands/releases_merge_names.ts index 66dbc69..54dc757 100644 --- a/commands/releases_merge_names.ts +++ b/commands/releases_merge_names.ts @@ -20,16 +20,20 @@ export default class ReleasesMergeNames extends BaseCommand { async run() { const merger = new Par2Merger() - let before = '9999-12-31' + // Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text + // so microseconds survive the round trip. + let before = ['9999-12-31', ''] let looked = 0 let merges = 0 let releases = 0 const examples: string[] = [] while (this.sets === undefined || looked < this.sets) { const { rows } = await db.rawQuery( - `SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE s.created_at < ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id) - GROUP BY s.set_id ORDER BY at DESC LIMIT ${this.dryRun ? 50 : 500}`, - [before] + `SELECT set_id, at::text AS at FROM ( + SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id) + GROUP BY s.set_id) x + WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`, + before ) if (rows.length === 0) break for (const { set_id: setId } of rows) { @@ -52,7 +56,7 @@ export default class ReleasesMergeNames extends BaseCommand { } } } - before = rows.at(-1).at + before = [rows.at(-1).at, rows.at(-1).set_id] if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`) } this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)