releases:merge-names: keyset paging on (created_at, set_id)
Sets stored in one batch share created_at, so paging on created_at < last skipped ~10k of 61k sets; the timestamp now also stays text so microseconds survive the round trip. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66b0980f4
commit
2dd5b4cd81
1 file changed
+9
-5
@@ -20,16 +20,20 @@ export default class ReleasesMergeNames extends BaseCommand {
|
||||
|
||||
async run() {
|
||||
const merger = new Par2Merger()
|
||||
let before = '9999-12-31'
|
||||
// Keyset on (newest created_at, set_id): sets stored in one batch share created_at, and the timestamp stays text
|
||||
// so microseconds survive the round trip.
|
||||
let before = ['9999-12-31', '']
|
||||
let looked = 0
|
||||
let merges = 0
|
||||
let releases = 0
|
||||
const examples: string[] = []
|
||||
while (this.sets === undefined || looked < this.sets) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE s.created_at < ? AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
GROUP BY s.set_id ORDER BY at DESC LIMIT ${this.dryRun ? 50 : 500}`,
|
||||
[before]
|
||||
`SELECT set_id, at::text AS at FROM (
|
||||
SELECT s.set_id, max(s.created_at) AS at FROM par2_sets s WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||
GROUP BY s.set_id) x
|
||||
WHERE (at, set_id) < (?::timestamptz, ?) ORDER BY at DESC, set_id DESC LIMIT ${this.dryRun ? 50 : 500}`,
|
||||
before
|
||||
)
|
||||
if (rows.length === 0) break
|
||||
for (const { set_id: setId } of rows) {
|
||||
@@ -52,7 +56,7 @@ export default class ReleasesMergeNames extends BaseCommand {
|
||||
}
|
||||
}
|
||||
}
|
||||
before = rows.at(-1).at
|
||||
before = [rows.at(-1).at, rows.at(-1).set_id]
|
||||
if (this.dryRun || looked % 20_000 < 500) this.logger.info(`${looked} sets looked at: ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
}
|
||||
this.logger.info(`Done: ${looked} sets looked at, ${releases} releases ${this.dryRun ? 'would merge' : 'merged'} into ${merges}.`)
|
||||
|
||||
Reference in new issue
Block a user