Keep binaries until inspected; seeding writes binaries only for inspection
Test and deploy / test (push) Successful in 1m24s
Test and deploy / deploy (push) Successful in 6m43s

Par2 naming needs the fingerprint inspection stores on a binary, and the
GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before
the inspection queue reached them. binaries.awaiting_inspection protects
them for up to 7 days; the GC deletes use partial indexes so they don't
read past waiting rows. Seeding no longer creates binaries for files that
aren't inspected (~85% of a.b.ath), which only churned the table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 20:03:43 +02:00
1 parent cc861ce8ae
commit b2cc75fdd0
11 files changed
+149 -35

No files matched your search

+6 -3
View File
@@ -99,6 +99,7 @@ class BinaryFileService {
const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
if (id) fileIdOf.set(Number(b.id), id)
}
const fresher = new Map(fresh.map((b) => [b.file_key, b]))
result.saved = inserted.length
result.existing = fileIdOf.size - inserted.length
@@ -110,9 +111,12 @@ class BinaryFileService {
`SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
[linked.map((b) => b.id)]
)
// Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
await trx.rawQuery(
`UPDATE binaries b SET file_id = v.file_id FROM unnest(?::bigint[], ?::bigint[]) AS v(id, file_id) WHERE b.id = v.id`,
[linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!)]
`UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
[linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
)
})
// Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
@@ -120,7 +124,6 @@ class BinaryFileService {
}
// As for one file before: an SFV posted earlier may list them (D34), then inspection.
const fresher = new Map(fresh.map((b) => [b.file_key, b]))
const added = inserted.flatMap((row) => {
const b = row.file_key ? fresher.get(row.file_key) : undefined
return b