Par2 size matching 100x faster; release bookkeeping once per statement
matchPar2Releases checked every same-size release_files row's set and date (18 s per batch of body-derived binaries); now an index on release_files (release_id, size) drops the repeated sizes first: 0.16 s. File inserts updated their release in four row triggers; one statement trigger now updates each release once per INSERT. Completeness is noted in release_dirty and recomputed by gc:scheduler every 30 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
288a6bab8d
commit
be2112f4ad
9 files changed
+208
-11
No files matched your search
@@ -272,18 +272,27 @@ class IngestService {
|
||||
*/
|
||||
async matchPar2Releases(binaryIds: number[]) {
|
||||
if (binaryIds.length === 0) return
|
||||
// Listed-once first (index-only on release_files (release_id, size): most candidate
|
||||
// sizes are a set's equal-sized volumes), the release's date only for what's left;
|
||||
// OFFSET 0 keeps that order. Was: every size match's date and count, 18 s per batch.
|
||||
await db.rawQuery(
|
||||
`UPDATE binaries b SET release_id = m.release_id
|
||||
FROM (
|
||||
SELECT b2.id, min(rf.release_id) AS release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size
|
||||
JOIN releases r ON r.id = rf.release_id
|
||||
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
|
||||
AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
|
||||
GROUP BY b2.id
|
||||
HAVING count(DISTINCT rf.release_id) = 1
|
||||
SELECT c.id, min(c.release_id) AS release_id
|
||||
FROM (
|
||||
SELECT b2.id, b2.posted_at, rf.release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size
|
||||
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM release_files x
|
||||
WHERE x.release_id = rf.release_id AND x.size = rf.size AND x.par2_file_id <> rf.par2_file_id)
|
||||
OFFSET 0
|
||||
) c
|
||||
JOIN releases r ON r.id = c.release_id
|
||||
WHERE abs(extract(epoch FROM (r.posted_at - c.posted_at))) < ?
|
||||
GROUP BY c.id
|
||||
HAVING count(DISTINCT c.release_id) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
|
||||
|
||||
Reference in new issue
Block a user