files:move-details: short batches, so files deleted meanwhile don't fail the copy
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fad1716d3a
commit
7c43abd230
1 file changed
+24
-4
@@ -2,6 +2,8 @@ import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
const BATCH = 50_000
|
||||
|
||||
/**
|
||||
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
|
||||
* listings still on `files` into file_details, a partition at a time.
|
||||
@@ -23,13 +25,31 @@ export default class FilesMoveDetails extends BaseCommand {
|
||||
let total = 0
|
||||
for (const { relname } of partitions) {
|
||||
const started = Date.now()
|
||||
const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`)
|
||||
if (rows[0].lo === null) continue
|
||||
let copied = 0
|
||||
// Short statements in id order: one statement over a whole partition took 30 minutes, and the
|
||||
// garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)).
|
||||
for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_details (file_id, content_type, archive)
|
||||
SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL
|
||||
ON CONFLICT (file_id) DO NOTHING`
|
||||
SELECT id, content_type, archive FROM ${relname}
|
||||
WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL)
|
||||
ON CONFLICT (file_id) DO NOTHING`,
|
||||
[lo, lo + BATCH]
|
||||
)
|
||||
total += result.rowCount ?? 0
|
||||
this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
copied += result.rowCount ?? 0
|
||||
break
|
||||
} catch (error: any) {
|
||||
// A file deleted while this batch ran: the batch again, without it.
|
||||
if (error.code !== '23503' || attempt >= 5) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
total += copied
|
||||
this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
this.logger.info(`${total} files' details copied.`)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user