diff --git a/commands/FilesMoveDetails.ts b/commands/FilesMoveDetails.ts index 1b95d58..729c03d 100644 --- a/commands/FilesMoveDetails.ts +++ b/commands/FilesMoveDetails.ts @@ -2,6 +2,8 @@ import { BaseCommand } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import db from '@adonisjs/lucid/services/db' +const BATCH = 50_000 + /** * One-off (LOG.md 2026-10-08 (14)): copies the content types and archive * listings still on `files` into file_details, a partition at a time. @@ -23,13 +25,31 @@ export default class FilesMoveDetails extends BaseCommand { let total = 0 for (const { relname } of partitions) { const started = Date.now() - const result = await db.rawQuery( - `INSERT INTO file_details (file_id, content_type, archive) - SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL - ON CONFLICT (file_id) DO NOTHING` - ) - total += result.rowCount ?? 0 - this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`) + const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`) + if (rows[0].lo === null) continue + let copied = 0 + // Short statements in id order: one statement over a whole partition took 30 minutes, and the + // garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)). + for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) { + for (let attempt = 1; ; attempt++) { + try { + const result = await db.rawQuery( + `INSERT INTO file_details (file_id, content_type, archive) + SELECT id, content_type, archive FROM ${relname} + WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL) + ON CONFLICT (file_id) DO NOTHING`, + [lo, lo + BATCH] + ) + copied += result.rowCount ?? 0 + break + } catch (error: any) { + // A file deleted while this batch ran: the batch again, without it. + if (error.code !== '23503' || attempt >= 5) throw error + } + } + } + total += copied + this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`) } this.logger.info(`${total} files' details copied.`) }