Inspection backlog in Postgres; BullMQ gets a working set

5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single
thread. Producers now add file ids to inspection_queue; index-scheduler
feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis
moves the existing backlog over once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/InspectionQueue.ts

@@ -0,0 +1,66 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import QueueService from '#services/QueueService'
+
+/** Files per transaction when moving from the table to BullMQ. */
+const FEED_BATCH = 5000
+
+/** Priorities: lower goes first. */
+export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
+
+/**
+ * The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
+ * Producers add file ids to the inspection_queue table; feed() moves them
+ * into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
+ * wait there, where worker:collection and inspecting nodes take them as
+ * before. Redis holds a working set instead of a backlog of days.
+ */
+class InspectionQueue {
+  readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
+
+  /** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
+  async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
+    const ids = [...new Set(fileIds)]
+    for (let i = 0; i < ids.length; i += 10_000) {
+      await db.rawQuery(
+        `INSERT INTO inspection_queue (file_id, priority)
+         SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
+         ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
+        [priority, ids.slice(i, i + 10_000)]
+      )
+    }
+  }
+
+  /** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
+  async feed(): Promise<number> {
+    const queue = QueueService.collectionQueue
+    const counts = await queue.getJobCounts('wait', 'prioritized')
+    let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
+    let moved = 0
+    while (room > 0) {
+      const n = await db.transaction(async (trx) => {
+        const { rows } = await trx.rawQuery(
+          `SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
+          [Math.min(room, FEED_BATCH)]
+        )
+        if (rows.length === 0) return 0
+        const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
+        // Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
+        await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
+        await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
+        return ids.length
+      })
+      if (n === 0) break
+      moved += n
+      room -= n
+    }
+    return moved
+  }
+
+  async size(): Promise<number> {
+    const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
+    return Number(rows[0].n)
+  }
+}
+
+export default new InspectionQueue()