Par2 size matching 100x faster; release bookkeeping once per statement
Test and deploy / test (push) Successful in 1m33s
Test and deploy / deploy (push) Failing after 1m29s

matchPar2Releases checked every same-size release_files row's set and
date (18 s per batch of body-derived binaries); now an index on
release_files (release_id, size) drops the repeated sizes first: 0.16 s.

File inserts updated their release in four row triggers; one statement
trigger now updates each release once per INSERT. Completeness is noted
in release_dirty and recomputed by gc:scheduler every 30 minutes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 13:48:59 +02:00
1 parent 288a6bab8d
commit be2112f4ad
9 files changed
+208 -11

No files matched your search

+18 -9
View File
@@ -272,18 +272,27 @@ class IngestService {
*/
async matchPar2Releases(binaryIds: number[]) {
if (binaryIds.length === 0) return
// Listed-once first (index-only on release_files (release_id, size): most candidate
// sizes are a set's equal-sized volumes), the release's date only for what's left;
// OFFSET 0 keeps that order. Was: every size match's date and count, 18 s per batch.
await db.rawQuery(
`UPDATE binaries b SET release_id = m.release_id
FROM (
SELECT b2.id, min(rf.release_id) AS release_id
FROM binaries b2
JOIN release_files rf ON rf.size = b2.size
JOIN releases r ON r.id = rf.release_id
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
GROUP BY b2.id
HAVING count(DISTINCT rf.release_id) = 1
SELECT c.id, min(c.release_id) AS release_id
FROM (
SELECT b2.id, b2.posted_at, rf.release_id
FROM binaries b2
JOIN release_files rf ON rf.size = b2.size
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
AND NOT EXISTS (
SELECT 1 FROM release_files x
WHERE x.release_id = rf.release_id AND x.size = rf.size AND x.par2_file_id <> rf.par2_file_id)
OFFSET 0
) c
JOIN releases r ON r.id = c.release_id
WHERE abs(extract(epoch FROM (r.posted_at - c.posted_at))) < ?
GROUP BY c.id
HAVING count(DISTINCT c.release_id) = 1
) m
WHERE b.id = m.id`,
[binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]