Par2 size matching 100x faster; release bookkeeping once per statement
matchPar2Releases checked every same-size release_files row's set and date (18 s per batch of body-derived binaries); now an index on release_files (release_id, size) drops the repeated sizes first: 0.16 s. File inserts updated their release in four row triggers; one statement trigger now updates each release once per INSERT. Completeness is noted in release_dirty and recomputed by gc:scheduler every 30 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
288a6bab8d
commit
be2112f4ad
9 files changed
+208
-11
No files matched your search
@@ -9,6 +9,7 @@ import IndexMaintenance from '#services/IndexMaintenance'
|
||||
import ReleaseMergeService from '#services/ReleaseMergeService'
|
||||
import StagingService from '#services/StagingService'
|
||||
import FilePartitions from '#services/FilePartitions'
|
||||
import ReleaseCompleteness from '#services/ReleaseCompleteness'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
|
||||
const CLEAN_BATCH = 1000
|
||||
@@ -30,6 +31,9 @@ export default class GcScheduler extends BaseCommand {
|
||||
@flags.boolean({ description: 'Report what would be removed without changing anything' })
|
||||
declare dryRun: boolean
|
||||
|
||||
/** When release completeness was last recomputed (it runs less often than the sweep). */
|
||||
private completenessAt = 0
|
||||
|
||||
public async run() {
|
||||
this.logger.info(
|
||||
`GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
|
||||
@@ -42,6 +46,11 @@ export default class GcScheduler extends BaseCommand {
|
||||
} catch (err: any) {
|
||||
this.logger.error(`GC sweep failed: ${err.message}`)
|
||||
}
|
||||
try {
|
||||
await this.releaseCompleteness()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Release completeness failed: ${err.message}`)
|
||||
}
|
||||
if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
|
||||
} while (!this.once && !Shutdown.requested)
|
||||
} finally {
|
||||
@@ -108,6 +117,14 @@ export default class GcScheduler extends BaseCommand {
|
||||
}
|
||||
|
||||
/** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
|
||||
private async releaseCompleteness() {
|
||||
if (this.dryRun || (!this.once && Date.now() - this.completenessAt < gcConfig.releaseCompleteIntervalMs)) return
|
||||
this.completenessAt = Date.now()
|
||||
const started = Date.now()
|
||||
const checked = await ReleaseCompleteness.sweep()
|
||||
if (checked > 0) this.logger.info(`Release completeness: ${checked} changed releases checked in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
|
||||
private async rebuildIndexes() {
|
||||
const key = 'gc:reindex:last'
|
||||
const last = Number((await RedisService.client.get(key)) ?? 0)
|
||||
|
||||
Reference in new issue
Block a user