Inspection backlog in Postgres; BullMQ gets a working set
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e7e338213a
commit
35d0b2da65
11 files changed
+219
-24
No files matched your search
+10
-13
@@ -4,6 +4,7 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import QueueService from '#services/QueueService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
|
||||
|
||||
const BATCH = 1000
|
||||
|
||||
@@ -71,19 +72,15 @@ export default class ContentBackfill extends BaseCommand {
|
||||
const jobs = files
|
||||
.filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
|
||||
.slice(0, this.limit ? this.limit - queued : undefined)
|
||||
.map((f) => ({
|
||||
name: 'process-collection',
|
||||
data: { fileId: Number(f.id) },
|
||||
// A new id per run: BullMQ ignores an add() for an id it still knows.
|
||||
// Targeted backfills go ahead of whatever is waiting.
|
||||
opts: {
|
||||
jobId: `content-${f.id}-${Date.now()}`,
|
||||
lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
|
||||
// niconico captures are many (~190k) and only cosmetic: behind everything else.
|
||||
...(this.nicovideo ? { priority: 10 } : {}),
|
||||
},
|
||||
}))
|
||||
if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
|
||||
.map((f) => Number(f.id))
|
||||
// Into the inspection backlog (InspectionQueue): targeted backfills go ahead of what's waiting;
|
||||
// niconico captures are many (~190k) and only cosmetic, so behind everything else.
|
||||
const priority = this.nicovideo
|
||||
? INSPECTION_PRIORITY.low
|
||||
: this.archiveNzbs || this.sevenZip || this.archives || this.sfv
|
||||
? INSPECTION_PRIORITY.targeted
|
||||
: INSPECTION_PRIORITY.normal
|
||||
if (jobs.length > 0) await InspectionQueue.add(jobs, priority)
|
||||
queued += jobs.length
|
||||
}
|
||||
this.logger.info(`Queued ${queued} files for content inspection.`)
|
||||
|
||||
Reference in new issue
Block a user