Inspection backlog in Postgres; BullMQ gets a working set

5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single
thread. Producers now add file ids to inspection_queue; index-scheduler
feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis
moves the existing backlog over once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/inspection_queue.spec.ts

@@ -0,0 +1,36 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import QueueService from '#services/QueueService'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
+
+test.group('Inspection queue', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.teardown(() => QueueService.closeAll())
+
+  test('the backlog stays in Postgres; BullMQ gets what fits, in priority order', async ({ assert }) => {
+    const queue = QueueService.collectionQueue
+    await queue.drain()
+    await InspectionQueue.add([9_000_001, 9_000_002], INSPECTION_PRIORITY.low)
+    await InspectionQueue.add([9_000_003])
+    await InspectionQueue.add([9_000_004], INSPECTION_PRIORITY.targeted)
+    // Queued again with a higher priority: one row, the higher priority.
+    await InspectionQueue.add([9_000_001], INSPECTION_PRIORITY.targeted)
+    assert.equal(await InspectionQueue.size(), 4)
+
+    const max = InspectionQueue.feedMax
+    ;(InspectionQueue as any).feedMax = 2
+    try {
+      assert.equal(await InspectionQueue.feed(), 2)
+      const waiting = (await queue.getWaiting()).map((job) => job.data.fileId).sort()
+      assert.deepEqual(waiting, [9_000_001, 9_000_004])
+      // Full: nothing more until the workers take some.
+      assert.equal(await InspectionQueue.feed(), 0)
+      const left = (await db.from('inspection_queue').select('file_id')).map((r) => Number(r.file_id)).sort()
+      assert.deepEqual(left, [9_000_002, 9_000_003])
+    } finally {
+      ;(InspectionQueue as any).feedMax = max
+      await queue.drain()
+    }
+  })
+})