Seed batches take file ids from the insert; merge split uploads hourly
Test and deploy / test (push) Successful in 1m29s
Test and deploy / deploy (push) Successful in 1m58s

SeedService looked up every file of a batch across all files partitions
after inserting it (15% of Postgres' active time); the merge job read
every release on each GC sweep.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 23:09:04 +02:00
1 parent b2cc75fdd0
commit 23bbdcc5ee
4 files changed
+24 -7

No files matched your search

+11 -4
View File
@@ -58,9 +58,16 @@ class SeedService {
result.files = inserted.length
result.existing = rows.length - inserted.length
// File ids for the binaries below (files queued for inspection only): new ones from the insert,
// existing ones looked up. Looking up every key of the batch probed all files partitions again,
// 15% of Postgres' active time (LOG.md 2026-10-06 (12)).
const fileIds = new Map<string, number>()
for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) {
fileIds.set(row.file_key, Number(row.id))
for (const row of inserted) if (row.file_key) fileIds.set(row.file_key, row.id)
const missing = files.filter((file) => !fileIds.has(file.fileKey) && ContentService.isCandidate(file.name)).map((file) => file.fileKey)
for (let i = 0; i < missing.length; i += 5000) {
for (const row of await db.from('files').whereIn('file_key', missing.slice(i, i + 5000)).select('id', 'file_key')) {
fileIds.set(row.file_key, Number(row.id))
}
}
// A completed binary, as worker:file would have left behind, only for files queued for content
// inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
@@ -97,9 +104,9 @@ class SeedService {
}
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
const rowByKey = new Map(rows.map((row) => [row.file_key, row]))
const added = inserted.flatMap((file) => {
const row = rowById.get(file.id)
const row = file.file_key ? rowByKey.get(file.file_key) : undefined
return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
})
await SfvService.claimAll(added)