Inspection backlog in Postgres; BullMQ gets a working set
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e7e338213a
commit
35d0b2da65
11 files changed
+219
-24
No files matched your search
@@ -3,8 +3,8 @@ import IngestService from '#services/IngestService'
|
||||
import FileStore, { insertLimits } from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
|
||||
/** Articles from a range's edge within which a file may continue in the neighbouring range. */
|
||||
const EDGE_ARTICLES = 50_000
|
||||
@@ -112,8 +112,8 @@ class SeedService {
|
||||
await SfvService.claimAll(added)
|
||||
const candidates = added
|
||||
.filter((file) => ContentService.isCandidate(file.name))
|
||||
.map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
|
||||
if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates)
|
||||
.map((file) => file.id)
|
||||
if (candidates.length > 0) await InspectionQueue.add(candidates)
|
||||
result.inspections = candidates.length
|
||||
return result
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user