Keep binaries until inspected; seeding writes binaries only for inspection
Test and deploy / test (push) Successful in 1m24s
Test and deploy / deploy (push) Successful in 6m43s

Par2 naming needs the fingerprint inspection stores on a binary, and the
GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before
the inspection queue reached them. binaries.awaiting_inspection protects
them for up to 7 days; the GC deletes use partial indexes so they don't
read past waiting rows. Seeding no longer creates binaries for files that
aren't inspected (~85% of a.b.ath), which only churned the table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 20:03:43 +02:00
1 parent cc861ce8ae
commit b2cc75fdd0
11 files changed
+149 -35

No files matched your search

+27 -16
View File
@@ -62,27 +62,38 @@ class SeedService {
for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) {
fileIds.set(row.file_key, Number(row.id))
}
// The completed binary worker:file would have left behind; an existing one just learns its file.
const binaries = files.map((file, i) => ({
file_key: file.fileKey,
poster: file.poster,
subject: file.subject,
name: file.name,
total_parts: file.totalParts,
groups: rows[i].groups,
posted_at: postedOf(file) ?? null,
first_seen_at: now,
last_seen_at: now,
completed_at: now,
release_id: rows[i].release_id,
file_id: fileIds.get(file.fileKey) ?? null,
}))
// A completed binary, as worker:file would have left behind, only for files queued for content
// inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
// (~85% in a.b.ath) it was only GC churn: 1.6M rows an hour inserted and deleted (LOG.md 2026-10-06 (11)).
const insertedKeys = new Set(inserted.map((row) => row.file_key))
const binaries = files.flatMap((file, i) => {
if (!ContentService.isCandidate(file.name)) return []
return [{
file_key: file.fileKey,
poster: file.poster,
subject: file.subject,
name: file.name,
total_parts: file.totalParts,
groups: rows[i].groups,
posted_at: postedOf(file) ?? null,
first_seen_at: now,
last_seen_at: now,
completed_at: now,
release_id: rows[i].release_id,
file_id: fileIds.get(file.fileKey) ?? null,
// New files are queued for inspection below; an existing file was queued when it was added.
awaiting_inspection: insertedKeys.has(file.fileKey) ? true : null,
}]
})
binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
for (let i = 0; i < binaries.length; i += insertLimits.rows) {
await knex('binaries')
.insert(binaries.slice(i, i + insertLimits.rows))
.onConflict('file_key')
.merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
.merge({
file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any,
awaiting_inspection: knex.raw('coalesce(binaries.awaiting_inspection, excluded.awaiting_inspection)') as any,
})
}
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.