files:move-details: short batches, so files deleted meanwhile don't fail the copy
Test and deploy / test (push) Successful in 1m15s
Test and deploy / deploy (push) Successful in 1m47s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-08 23:08:31 +02:00
1 parent fad1716d3a
commit 7c43abd230
1 file changed
+27 -7
+27 -7
View File
@@ -2,6 +2,8 @@ import { BaseCommand } from '@adonisjs/core/ace'
import { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
const BATCH = 50_000
/**
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
* listings still on `files` into file_details, a partition at a time.
@@ -23,13 +25,31 @@ export default class FilesMoveDetails extends BaseCommand {
let total = 0
for (const { relname } of partitions) {
const started = Date.now()
const result = await db.rawQuery(
`INSERT INTO file_details (file_id, content_type, archive)
SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL
ON CONFLICT (file_id) DO NOTHING`
)
total += result.rowCount ?? 0
this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`)
if (rows[0].lo === null) continue
let copied = 0
// Short statements in id order: one statement over a whole partition took 30 minutes, and the
// garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)).
for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) {
for (let attempt = 1; ; attempt++) {
try {
const result = await db.rawQuery(
`INSERT INTO file_details (file_id, content_type, archive)
SELECT id, content_type, archive FROM ${relname}
WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL)
ON CONFLICT (file_id) DO NOTHING`,
[lo, lo + BATCH]
)
copied += result.rowCount ?? 0
break
} catch (error: any) {
// A file deleted while this batch ran: the batch again, without it.
if (error.code !== '23503' || attempt >= 5) throw error
}
}
}
total += copied
this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
}
this.logger.info(`${total} files' details copied.`)
}