Daan MeijerandClaude Opus 5.5
f0ff8e200f
Copying all existing keys in the migration read the files rows from the HDD for over half an hour and held up the deploy. gc:scheduler now copies them in slices (FILE_KEYS_FILL_SECONDS per sweep) and FileStore checks every partition until the fill is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
2.5 KiB
TypeScript
61 lines
2.5 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
|
|
/** files ids per slice: a few seconds of reads from the HDD. */
|
|
const SLICE = 100_000
|
|
|
|
/**
|
|
* The file_keys filter (LOG.md 2026-10-06 (7)): a 64-bit hash of every
|
|
* file_key. The insert trigger adds new keys; `fill` copies the keys of
|
|
* files from before, in id order and a slice at a time, and marks the
|
|
* filter complete when it reaches the files that existed when it started
|
|
* (everything after that has been added by the trigger, which was live
|
|
* first). FileStore uses the filter only once it is complete.
|
|
*/
|
|
class FileKeys {
|
|
private completeSince = 0
|
|
|
|
/** Whether every file's key is in file_keys; cached once true. */
|
|
async complete(): Promise<boolean> {
|
|
if (this.completeSince) return true
|
|
const row = await db.from('file_keys_fill').where('id', 1).first()
|
|
if (row?.complete) this.completeSince = Date.now()
|
|
return Boolean(row?.complete)
|
|
}
|
|
|
|
/** Forgets the cached state (tests). */
|
|
forget() {
|
|
this.completeSince = 0
|
|
}
|
|
|
|
/** Copies keys until `budgetMs` has passed or the fill is done; returns the files read and whether it's complete. */
|
|
async fill(budgetMs: number): Promise<{ read: number; complete: boolean; nextId: number; untilId: number }> {
|
|
const started = Date.now()
|
|
let state = await db.from('file_keys_fill').where('id', 1).first()
|
|
if (!state) return { read: 0, complete: false, nextId: 0, untilId: 0 }
|
|
if (state.until_id === null) {
|
|
// Everything inserted from here on goes through the trigger (live since the migration).
|
|
const [{ max }] = await db.from('files').max('id as max')
|
|
await db.from('file_keys_fill').where('id', 1).update({ until_id: Number(max ?? 0), updated_at: new Date() })
|
|
state = await db.from('file_keys_fill').where('id', 1).first()
|
|
}
|
|
let next = Number(state.next_id)
|
|
const until = Number(state.until_id)
|
|
let read = 0
|
|
while (next <= until && Date.now() - started < budgetMs) {
|
|
const to = Math.min(until, next + SLICE - 1)
|
|
const result = await db.rawQuery(
|
|
`INSERT INTO file_keys (key_hash)
|
|
SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE id BETWEEN ? AND ? AND file_key IS NOT NULL
|
|
ON CONFLICT DO NOTHING`,
|
|
[next, to]
|
|
)
|
|
read += Number(result.rowCount ?? 0)
|
|
next = to + 1
|
|
await db.from('file_keys_fill').where('id', 1).update({ next_id: next, complete: next > until, updated_at: new Date() })
|
|
}
|
|
return { read, complete: next > until, nextId: next, untilId: until }
|
|
}
|
|
}
|
|
|
|
export default new FileKeys()
|