Keep binaries until inspected; seeding writes binaries only for inspection
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cc861ce8ae
commit
b2cc75fdd0
11 files changed
+149
-35
No files matched your search
+20
-11
@@ -178,19 +178,27 @@ export default class GcScheduler extends BaseCommand {
|
||||
* already live on in `files`; conflicted ones will never become a file.
|
||||
* Completed ones still waiting for worker:file are kept.
|
||||
*/
|
||||
/**
|
||||
* Binaries without new segments for staleAfterSeconds: incomplete ones are given up on,
|
||||
* finished ones have their file. A finished one whose file waits for content inspection
|
||||
* stays until it's inspected (its fingerprint names it from par2 sets, D16), at most
|
||||
* awaitInspectionSeconds (LOG.md 2026-10-06 (11)). Three deletes, one per partial index,
|
||||
* so none reads past millions of waiting rows.
|
||||
*/
|
||||
private async deleteStaleBinaries(): Promise<number> {
|
||||
const query = db
|
||||
.from('binaries')
|
||||
.where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
|
||||
.where((q) =>
|
||||
q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
|
||||
)
|
||||
|
||||
if (this.dryRun) {
|
||||
const [{ count }] = await query.count('* as count')
|
||||
return Number(count)
|
||||
const stale = new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000)
|
||||
const waited = new Date(Date.now() - gcConfig.binaries.awaitInspectionSeconds * 1000)
|
||||
const deletes = [
|
||||
db.from('binaries').where('last_seen_at', '<', stale).whereNotNull('file_id').whereRaw('awaiting_inspection IS NOT TRUE'),
|
||||
db.from('binaries').where('last_seen_at', '<', stale).where((q) => q.whereNull('completed_at').orWhereNotNull('conflicted_at')),
|
||||
db.from('binaries').where('last_seen_at', '<', waited).whereRaw('awaiting_inspection IS TRUE'),
|
||||
]
|
||||
let total = 0
|
||||
for (const query of deletes) {
|
||||
if (this.dryRun) total += Number((await query.count('* as count'))[0].count)
|
||||
else total += Number(await query.delete())
|
||||
}
|
||||
return Number(await query.delete())
|
||||
return total
|
||||
}
|
||||
|
||||
private async deleteOldDeferredFiles(): Promise<number> {
|
||||
@@ -283,6 +291,7 @@ export default class GcScheduler extends BaseCommand {
|
||||
SELECT id FROM binaries
|
||||
WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
|
||||
AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
|
||||
AND awaiting_inspection IS NOT TRUE
|
||||
ORDER BY last_seen_at LIMIT ?)`,
|
||||
[cutoff, gcConfig.deleteBatch]
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user