Daan MeijerandClaude Opus 5.5
35d0b2da65
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
37 lines
1.6 KiB
TypeScript
37 lines
1.6 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import QueueService from '#services/QueueService'
|
|
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
|
|
|
|
test.group('Inspection queue', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
group.teardown(() => QueueService.closeAll())
|
|
|
|
test('the backlog stays in Postgres; BullMQ gets what fits, in priority order', async ({ assert }) => {
|
|
const queue = QueueService.collectionQueue
|
|
await queue.drain()
|
|
await InspectionQueue.add([9_000_001, 9_000_002], INSPECTION_PRIORITY.low)
|
|
await InspectionQueue.add([9_000_003])
|
|
await InspectionQueue.add([9_000_004], INSPECTION_PRIORITY.targeted)
|
|
// Queued again with a higher priority: one row, the higher priority.
|
|
await InspectionQueue.add([9_000_001], INSPECTION_PRIORITY.targeted)
|
|
assert.equal(await InspectionQueue.size(), 4)
|
|
|
|
const max = InspectionQueue.feedMax
|
|
;(InspectionQueue as any).feedMax = 2
|
|
try {
|
|
assert.equal(await InspectionQueue.feed(), 2)
|
|
const waiting = (await queue.getWaiting()).map((job) => job.data.fileId).sort()
|
|
assert.deepEqual(waiting, [9_000_001, 9_000_004])
|
|
// Full: nothing more until the workers take some.
|
|
assert.equal(await InspectionQueue.feed(), 0)
|
|
const left = (await db.from('inspection_queue').select('file_id')).map((r) => Number(r.file_id)).sort()
|
|
assert.deepEqual(left, [9_000_002, 9_000_003])
|
|
} finally {
|
|
;(InspectionQueue as any).feedMax = max
|
|
await queue.drain()
|
|
}
|
|
})
|
|
})
|