Keep binaries until inspected; seeding writes binaries only for inspection
Test and deploy / test (push) Successful in 1m24s
Test and deploy / deploy (push) Successful in 6m43s

Par2 naming needs the fingerprint inspection stores on a binary, and the
GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before
the inspection queue reached them. binaries.awaiting_inspection protects
them for up to 7 days; the GC deletes use partial indexes so they don't
read past waiting rows. Seeding no longer creates binaries for files that
aren't inspected (~85% of a.b.ath), which only churned the table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 20:03:43 +02:00
1 parent cc861ce8ae
commit b2cc75fdd0
11 files changed
+149 -35

No files matched your search

+20 -11
View File
@@ -178,19 +178,27 @@ export default class GcScheduler extends BaseCommand {
* already live on in `files`; conflicted ones will never become a file.
* Completed ones still waiting for worker:file are kept.
*/
/**
* Binaries without new segments for staleAfterSeconds: incomplete ones are given up on,
* finished ones have their file. A finished one whose file waits for content inspection
* stays until it's inspected (its fingerprint names it from par2 sets, D16), at most
* awaitInspectionSeconds (LOG.md 2026-10-06 (11)). Three deletes, one per partial index,
* so none reads past millions of waiting rows.
*/
private async deleteStaleBinaries(): Promise<number> {
const query = db
.from('binaries')
.where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
.where((q) =>
q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
)
if (this.dryRun) {
const [{ count }] = await query.count('* as count')
return Number(count)
const stale = new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000)
const waited = new Date(Date.now() - gcConfig.binaries.awaitInspectionSeconds * 1000)
const deletes = [
db.from('binaries').where('last_seen_at', '<', stale).whereNotNull('file_id').whereRaw('awaiting_inspection IS NOT TRUE'),
db.from('binaries').where('last_seen_at', '<', stale).where((q) => q.whereNull('completed_at').orWhereNotNull('conflicted_at')),
db.from('binaries').where('last_seen_at', '<', waited).whereRaw('awaiting_inspection IS TRUE'),
]
let total = 0
for (const query of deletes) {
if (this.dryRun) total += Number((await query.count('* as count'))[0].count)
else total += Number(await query.delete())
}
return Number(await query.delete())
return total
}
private async deleteOldDeferredFiles(): Promise<number> {
@@ -283,6 +291,7 @@ export default class GcScheduler extends BaseCommand {
SELECT id FROM binaries
WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
AND awaiting_inspection IS NOT TRUE
ORDER BY last_seen_at LIMIT ?)`,
[cutoff, gcConfig.deleteBatch]
)