Inspection backlog in Postgres; BullMQ gets a working set
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e7e338213a
commit
35d0b2da65
11 files changed
+219
-24
No files matched your search
@@ -7,9 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import FileStore from '#services/FileStore'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
|
||||
export interface NzbImportResult {
|
||||
releaseId: number
|
||||
@@ -207,9 +207,7 @@ class NzbImportService {
|
||||
// Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
|
||||
const archives = added.filter((f) => isArchiveStart(f.filename))
|
||||
if (archives.length) {
|
||||
await QueueService.collectionQueue.addBulk(
|
||||
archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
|
||||
)
|
||||
await InspectionQueue.add(archives.map((f) => Number(f.id)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user