file_keys: fill in the background, use it once complete
Copying all existing keys in the migration read the files rows from the HDD for over half an hour and held up the deploy. gc:scheduler now copies them in slices (FILE_KEYS_FILL_SECONDS per sweep) and FileStore checks every partition until the fill is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
693e6c3244
commit
f0ff8e200f
8 files changed
+163
-18
No files matched your search
@@ -10,6 +10,7 @@ import ReleaseMergeService from '#services/ReleaseMergeService'
|
||||
import StagingService from '#services/StagingService'
|
||||
import FilePartitions from '#services/FilePartitions'
|
||||
import ReleaseCompleteness from '#services/ReleaseCompleteness'
|
||||
import FileKeys from '#services/FileKeys'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
|
||||
const CLEAN_BATCH = 1000
|
||||
@@ -51,6 +52,11 @@ export default class GcScheduler extends BaseCommand {
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Release completeness failed: ${err.message}`)
|
||||
}
|
||||
try {
|
||||
await this.fillFileKeys()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`file_keys fill failed: ${err.message}`)
|
||||
}
|
||||
if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
|
||||
} while (!this.once && !Shutdown.requested)
|
||||
} finally {
|
||||
@@ -117,6 +123,17 @@ export default class GcScheduler extends BaseCommand {
|
||||
}
|
||||
|
||||
/** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
|
||||
/** The keys of files from before file_keys, a slice of time per sweep, until done. */
|
||||
private async fillFileKeys() {
|
||||
if (this.dryRun || (await FileKeys.complete())) return
|
||||
const started = Date.now()
|
||||
const { read, complete, nextId, untilId } = await FileKeys.fill(gcConfig.fileKeysFillMs)
|
||||
this.logger.info(
|
||||
`file_keys: ${read.toLocaleString('en')} keys copied in ${Math.round((Date.now() - started) / 1000)} s, ` +
|
||||
(complete ? 'complete: file inserts use the filter now.' : `at file ${nextId.toLocaleString('en')} of ${untilId.toLocaleString('en')}.`)
|
||||
)
|
||||
}
|
||||
|
||||
private async releaseCompleteness() {
|
||||
if (this.dryRun || (!this.once && Date.now() - this.completenessAt < gcConfig.releaseCompleteIntervalMs)) return
|
||||
this.completenessAt = Date.now()
|
||||
|
||||
Reference in new issue
Block a user