Daan MeijerandClaude Opus 5.5
35d0b2da65
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
67 lines
2.7 KiB
TypeScript
67 lines
2.7 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import env from '#start/env'
|
|
import QueueService from '#services/QueueService'
|
|
|
|
/** Files per transaction when moving from the table to BullMQ. */
|
|
const FEED_BATCH = 5000
|
|
|
|
/** Priorities: lower goes first. */
|
|
export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
|
|
|
|
/**
|
|
* The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
|
|
* Producers add file ids to the inspection_queue table; feed() moves them
|
|
* into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
|
|
* wait there, where worker:collection and inspecting nodes take them as
|
|
* before. Redis holds a working set instead of a backlog of days.
|
|
*/
|
|
class InspectionQueue {
|
|
readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
|
|
|
|
/** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
|
|
async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
|
|
const ids = [...new Set(fileIds)]
|
|
for (let i = 0; i < ids.length; i += 10_000) {
|
|
await db.rawQuery(
|
|
`INSERT INTO inspection_queue (file_id, priority)
|
|
SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
|
|
ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
|
|
[priority, ids.slice(i, i + 10_000)]
|
|
)
|
|
}
|
|
}
|
|
|
|
/** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
|
|
async feed(): Promise<number> {
|
|
const queue = QueueService.collectionQueue
|
|
const counts = await queue.getJobCounts('wait', 'prioritized')
|
|
let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
|
|
let moved = 0
|
|
while (room > 0) {
|
|
const n = await db.transaction(async (trx) => {
|
|
const { rows } = await trx.rawQuery(
|
|
`SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
|
|
[Math.min(room, FEED_BATCH)]
|
|
)
|
|
if (rows.length === 0) return 0
|
|
const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
|
|
// Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
|
|
await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
|
|
await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
|
|
return ids.length
|
|
})
|
|
if (n === 0) break
|
|
moved += n
|
|
room -= n
|
|
}
|
|
return moved
|
|
}
|
|
|
|
async size(): Promise<number> {
|
|
const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
|
|
return Number(rows[0].n)
|
|
}
|
|
}
|
|
|
|
export default new InspectionQueue()
|