93 lines
4.3 KiB
TypeScript
93 lines
4.3 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import { setTimeout as sleep } from 'node:timers/promises'
|
|
import { DiskSpace } from '#services/disk_space'
|
|
|
|
/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
|
|
const CURSOR = 'recategorize'
|
|
|
|
/**
|
|
* The categories of releases from before migration 5200 (release_category_for: the fixed name rules, the inner
|
|
* name, the groups; then the spots'), in batches by id: only rows whose category changes are written, each a new row
|
|
* version in a 70 GB table on the NVMe, so it waits while the filesystem is under --floor-gb free (autovacuum makes
|
|
* the dead versions reusable) and goes on from its cursor when run again.
|
|
*/
|
|
export default class ReleasesRecategorize extends BaseCommand {
|
|
static commandName = 'releases:recategorize'
|
|
static description = 'Recomputes release categories (name, inner name, groups, spots) in batches; --dry-run counts only'
|
|
static options: CommandOptions = { startApp: true }
|
|
|
|
@flags.number({ description: 'Release ids per batch' })
|
|
declare batch: number
|
|
|
|
@flags.number({ description: 'Stop after this id (a sample with --dry-run)' })
|
|
declare toId: number
|
|
|
|
@flags.number({ description: 'Wait while the NVMe has less free (GB)' })
|
|
declare floorGb: number
|
|
|
|
@flags.boolean({ description: 'Count what would change, write nothing' })
|
|
declare dryRun: boolean
|
|
|
|
@flags.boolean({ description: 'Start from the first id, not the cursor' })
|
|
declare restart: boolean
|
|
|
|
async run() {
|
|
const batch = this.batch ?? 20_000
|
|
const space = new DiskSpace('recategorizing', '/', this.floorGb ?? 14)
|
|
const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
|
|
const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
|
|
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
|
let after = this.restart || this.dryRun || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
|
|
const changes = new Map<string, number>()
|
|
let changed = 0
|
|
let looked = 0
|
|
const started = Date.now()
|
|
while (after < hi) {
|
|
while (!this.dryRun && (await space.short())) await sleep(60_000)
|
|
const to = Math.min(after + batch, hi)
|
|
const fresh = `release_category_for(name, inner_name, groups, NULL)`
|
|
if (this.dryRun) {
|
|
const { rows } = await db.rawQuery(
|
|
`SELECT category AS was, ${fresh} AS now, count(*)::int AS n FROM releases WHERE id > ? AND id <= ? GROUP BY 1, 2`,
|
|
[after, to]
|
|
)
|
|
for (const r of rows) {
|
|
looked += r.n
|
|
if (r.was === r.now) continue
|
|
changed += r.n
|
|
changes.set(`${r.was} → ${r.now}`, (changes.get(`${r.was} → ${r.now}`) ?? 0) + r.n)
|
|
}
|
|
} else {
|
|
const result = await db.rawQuery(
|
|
`UPDATE releases r SET category = x.c FROM (SELECT id, ${fresh} AS c FROM releases WHERE id > ? AND id <= ?) x
|
|
WHERE r.id = x.id AND r.category IS DISTINCT FROM x.c`,
|
|
[after, to]
|
|
)
|
|
changed += result.rowCount ?? 0
|
|
await db.rawQuery(
|
|
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
|
[CURSOR, String(to)]
|
|
)
|
|
}
|
|
after = to
|
|
if (Math.floor(after / 5_000_000) !== Math.floor((after - batch) / 5_000_000)) {
|
|
this.logger.info(`Up to id ${after} of ${hi}: ${changed} changed${this.dryRun ? ` of ${looked}` : ''} (${Math.round((Date.now() - started) / 1000)} s).`)
|
|
}
|
|
}
|
|
if (this.dryRun) {
|
|
this.logger.info(`Would change ${changed} of ${looked}:`)
|
|
for (const [change, n] of [...changes].sort((a, b) => b[1] - a[1]).slice(0, 30)) this.logger.info(` ${change}: ${n}`)
|
|
return
|
|
}
|
|
// The spots' categories for the linked releases whose names say nothing.
|
|
const spots = await db.rawQuery(
|
|
`UPDATE releases r SET category = spot_category(s.category, s.subcats)
|
|
FROM release_spots rs JOIN spots s ON s.message_id = rs.spot_id
|
|
WHERE r.id = rs.release_id AND r.category = 8010 AND spot_category(s.category, s.subcats) IS NOT NULL`
|
|
)
|
|
this.logger.info(`Done: ${changed} releases recategorized, ${spots.rowCount ?? 0} by their spots.`)
|
|
}
|
|
}
|