Keep binaries until inspected; seeding writes binaries only for inspection
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cc861ce8ae
commit
b2cc75fdd0
11 files changed
+149
-35
No files matched your search
+27
-16
@@ -62,27 +62,38 @@ class SeedService {
|
||||
for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) {
|
||||
fileIds.set(row.file_key, Number(row.id))
|
||||
}
|
||||
// The completed binary worker:file would have left behind; an existing one just learns its file.
|
||||
const binaries = files.map((file, i) => ({
|
||||
file_key: file.fileKey,
|
||||
poster: file.poster,
|
||||
subject: file.subject,
|
||||
name: file.name,
|
||||
total_parts: file.totalParts,
|
||||
groups: rows[i].groups,
|
||||
posted_at: postedOf(file) ?? null,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
completed_at: now,
|
||||
release_id: rows[i].release_id,
|
||||
file_id: fileIds.get(file.fileKey) ?? null,
|
||||
}))
|
||||
// A completed binary, as worker:file would have left behind, only for files queued for content
|
||||
// inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
|
||||
// (~85% in a.b.ath) it was only GC churn: 1.6M rows an hour inserted and deleted (LOG.md 2026-10-06 (11)).
|
||||
const insertedKeys = new Set(inserted.map((row) => row.file_key))
|
||||
const binaries = files.flatMap((file, i) => {
|
||||
if (!ContentService.isCandidate(file.name)) return []
|
||||
return [{
|
||||
file_key: file.fileKey,
|
||||
poster: file.poster,
|
||||
subject: file.subject,
|
||||
name: file.name,
|
||||
total_parts: file.totalParts,
|
||||
groups: rows[i].groups,
|
||||
posted_at: postedOf(file) ?? null,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
completed_at: now,
|
||||
release_id: rows[i].release_id,
|
||||
file_id: fileIds.get(file.fileKey) ?? null,
|
||||
// New files are queued for inspection below; an existing file was queued when it was added.
|
||||
awaiting_inspection: insertedKeys.has(file.fileKey) ? true : null,
|
||||
}]
|
||||
})
|
||||
binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
for (let i = 0; i < binaries.length; i += insertLimits.rows) {
|
||||
await knex('binaries')
|
||||
.insert(binaries.slice(i, i + insertLimits.rows))
|
||||
.onConflict('file_key')
|
||||
.merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
|
||||
.merge({
|
||||
file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any,
|
||||
awaiting_inspection: knex.raw('coalesce(binaries.awaiting_inspection, excluded.awaiting_inspection)') as any,
|
||||
})
|
||||
}
|
||||
|
||||
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
|
||||
|
||||
Reference in new issue
Block a user