Inspection backlog in Postgres; BullMQ gets a working set
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e7e338213a
commit
35d0b2da65
11 files changed
+219
-24
No files matched your search
@@ -5,6 +5,7 @@ import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
import SplitService from '#services/SplitService'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
@@ -108,6 +109,12 @@ export default class IndexScheduler extends BaseCommand {
|
||||
}
|
||||
}
|
||||
|
||||
// The inspection backlog lives in Postgres; BullMQ gets a working set (LOG.md 2026-10-07 (8)).
|
||||
const feedInspection = async () => {
|
||||
const moved = await InspectionQueue.feed()
|
||||
if (moved > 0) this.logger.info(`Inspection: ${moved} files moved from the backlog to the queue.`)
|
||||
}
|
||||
|
||||
const backfill = async () => {
|
||||
try {
|
||||
const queued = await BackfillService.tick()
|
||||
@@ -122,7 +129,7 @@ export default class IndexScheduler extends BaseCommand {
|
||||
// here once ended the loop while the process stayed up, and live indexing stopped for ~3 hours
|
||||
// (LOG.md 2026-10-07 (3)).
|
||||
while (!Shutdown.requested) {
|
||||
for (const step of [schedule, backfill, split]) {
|
||||
for (const step of [schedule, backfill, split, feedInspection]) {
|
||||
try {
|
||||
await step()
|
||||
} catch (err: any) {
|
||||
|
||||
Reference in new issue
Block a user