Keep binaries until inspected; seeding writes binaries only for inspection
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cc861ce8ae
commit
b2cc75fdd0
11 files changed
+149
-35
No files matched your search
@@ -99,6 +99,7 @@ class BinaryFileService {
|
||||
const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
|
||||
if (id) fileIdOf.set(Number(b.id), id)
|
||||
}
|
||||
const fresher = new Map(fresh.map((b) => [b.file_key, b]))
|
||||
result.saved = inserted.length
|
||||
result.existing = fileIdOf.size - inserted.length
|
||||
|
||||
@@ -110,9 +111,12 @@ class BinaryFileService {
|
||||
`SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
|
||||
[linked.map((b) => b.id)]
|
||||
)
|
||||
// Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
|
||||
const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET file_id = v.file_id FROM unnest(?::bigint[], ?::bigint[]) AS v(id, file_id) WHERE b.id = v.id`,
|
||||
[linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!)]
|
||||
`UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
|
||||
FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
|
||||
[linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
|
||||
)
|
||||
})
|
||||
// Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
|
||||
@@ -120,7 +124,6 @@ class BinaryFileService {
|
||||
}
|
||||
|
||||
// As for one file before: an SFV posted earlier may list them (D34), then inspection.
|
||||
const fresher = new Map(fresh.map((b) => [b.file_key, b]))
|
||||
const added = inserted.flatMap((row) => {
|
||||
const b = row.file_key ? fresher.get(row.file_key) : undefined
|
||||
return b
|
||||
|
||||
Reference in new issue
Block a user