file_keys: fill in the background, use it once complete
Test and deploy / test (push) Successful in 1m37s
Test and deploy / deploy (push) Successful in 2m23s

Copying all existing keys in the migration read the files rows from the
HDD for over half an hour and held up the deploy. gc:scheduler now copies
them in slices (FILE_KEYS_FILL_SECONDS per sweep) and FileStore checks
every partition until the fill is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 14:54:12 +02:00
1 parent 693e6c3244
commit f0ff8e200f
8 files changed
+163 -18

No files matched your search

+17
View File
@@ -10,6 +10,7 @@ import ReleaseMergeService from '#services/ReleaseMergeService'
import StagingService from '#services/StagingService'
import FilePartitions from '#services/FilePartitions'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
import FileKeys from '#services/FileKeys'
import Shutdown from '#services/Shutdown'
const CLEAN_BATCH = 1000
@@ -51,6 +52,11 @@ export default class GcScheduler extends BaseCommand {
} catch (err: any) {
this.logger.error(`Release completeness failed: ${err.message}`)
}
try {
await this.fillFileKeys()
} catch (err: any) {
this.logger.error(`file_keys fill failed: ${err.message}`)
}
if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
} while (!this.once && !Shutdown.requested)
} finally {
@@ -117,6 +123,17 @@ export default class GcScheduler extends BaseCommand {
}
/** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
/** The keys of files from before file_keys, a slice of time per sweep, until done. */
private async fillFileKeys() {
if (this.dryRun || (await FileKeys.complete())) return
const started = Date.now()
const { read, complete, nextId, untilId } = await FileKeys.fill(gcConfig.fileKeysFillMs)
this.logger.info(
`file_keys: ${read.toLocaleString('en')} keys copied in ${Math.round((Date.now() - started) / 1000)} s, ` +
(complete ? 'complete: file inserts use the filter now.' : `at file ${nextId.toLocaleString('en')} of ${untilId.toLocaleString('en')}.`)
)
}
private async releaseCompleteness() {
if (this.dryRun || (!this.once && Date.now() - this.completenessAt < gcConfig.releaseCompleteIntervalMs)) return
this.completenessAt = Date.now()