import db from '@adonisjs/lucid/services/db' import env from '#start/env' import QueueService from '#services/QueueService' /** Files per transaction when moving from the table to BullMQ. */ const FEED_BATCH = 5000 /** Priorities: lower goes first. */ export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const /** * The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)). * Producers add file ids to the inspection_queue table; feed() moves them * into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs * wait there, where worker:collection and inspecting nodes take them as * before. Redis holds a working set instead of a backlog of days. */ class InspectionQueue { readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000) /** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */ async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) { const ids = [...new Set(fileIds)] for (let i = 0; i < ids.length; i += 10_000) { await db.rawQuery( `INSERT INTO inspection_queue (file_id, priority) SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`, [priority, ids.slice(i, i + 10_000)] ) } } /** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */ async feed(): Promise { const queue = QueueService.collectionQueue const counts = await queue.getJobCounts('wait', 'prioritized') let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0) let moved = 0 while (room > 0) { const n = await db.transaction(async (trx) => { const { rows } = await trx.rawQuery( `SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`, [Math.min(room, FEED_BATCH)] ) if (rows.length === 0) return 0 const ids = rows.map((row: { file_id: string }) => Number(row.file_id)) // Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id). await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } }))) await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids]) return ids.length }) if (n === 0) break moved += n room -= n } return moved } async size(): Promise { const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`) return Number(rows[0].n) } } export default new InspectionQueue()