Inspection backlog in Postgres; BullMQ gets a working set

5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single
thread. Producers now add file ids to inspection_queue; index-scheduler
feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis
moves the existing backlog over once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/BinaryFileService.ts

@@ -1,8 +1,8 @@
 import db from '@adonisjs/lucid/services/db'
 import FileStore from '#services/FileStore'
 import SfvService from '#services/SfvService'
-import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
+import InspectionQueue from '#services/InspectionQueue'
 
 export interface BinaryFilesResult {
   /** New files. */
@@ -134,8 +134,8 @@ class BinaryFileService {
     // By the posted name, as before: inspection skips later volumes and readable media names anyway.
     const jobs = added
       .filter((file) => ContentService.isCandidate(file.posted))
-      .map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
-    if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+      .map((file) => file.id)
+    if (jobs.length > 0) await InspectionQueue.add(jobs)
     result.inspections = jobs.length
     return result
   }

app/services/InspectionQueue.ts

@@ -0,0 +1,66 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import QueueService from '#services/QueueService'
+
+/** Files per transaction when moving from the table to BullMQ. */
+const FEED_BATCH = 5000
+
+/** Priorities: lower goes first. */
+export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
+
+/**
+ * The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
+ * Producers add file ids to the inspection_queue table; feed() moves them
+ * into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
+ * wait there, where worker:collection and inspecting nodes take them as
+ * before. Redis holds a working set instead of a backlog of days.
+ */
+class InspectionQueue {
+  readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
+
+  /** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
+  async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
+    const ids = [...new Set(fileIds)]
+    for (let i = 0; i < ids.length; i += 10_000) {
+      await db.rawQuery(
+        `INSERT INTO inspection_queue (file_id, priority)
+         SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
+         ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
+        [priority, ids.slice(i, i + 10_000)]
+      )
+    }
+  }
+
+  /** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
+  async feed(): Promise<number> {
+    const queue = QueueService.collectionQueue
+    const counts = await queue.getJobCounts('wait', 'prioritized')
+    let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
+    let moved = 0
+    while (room > 0) {
+      const n = await db.transaction(async (trx) => {
+        const { rows } = await trx.rawQuery(
+          `SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
+          [Math.min(room, FEED_BATCH)]
+        )
+        if (rows.length === 0) return 0
+        const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
+        // Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
+        await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
+        await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
+        return ids.length
+      })
+      if (n === 0) break
+      moved += n
+      room -= n
+    }
+    return moved
+  }
+
+  async size(): Promise<number> {
+    const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
+    return Number(rows[0].n)
+  }
+}
+
+export default new InspectionQueue()

app/services/NzbImportService.ts

@@ -7,9 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
 import { parseSubject } from '#lib/subject/index'
 import { fileStem } from '#lib/release/index'
 import { looksRandom } from '#services/ArchiveService'
-import QueueService from '#services/QueueService'
 import FileStore from '#services/FileStore'
 import { isArchiveStart } from '#lib/content/index'
+import InspectionQueue from '#services/InspectionQueue'
 
 export interface NzbImportResult {
   releaseId: number
@@ -207,9 +207,7 @@ class NzbImportService {
     // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
     const archives = added.filter((f) => isArchiveStart(f.filename))
     if (archives.length) {
-      await QueueService.collectionQueue.addBulk(
-        archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
-      )
+      await InspectionQueue.add(archives.map((f) => Number(f.id)))
     }
     return result
   }

app/services/SeedService.ts

@@ -3,8 +3,8 @@ import IngestService from '#services/IngestService'
 import FileStore, { insertLimits } from '#services/FileStore'
 import SfvService from '#services/SfvService'
 import ContentService from '#services/ContentService'
-import QueueService from '#services/QueueService'
 import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
+import InspectionQueue from '#services/InspectionQueue'
 
 /** Articles from a range's edge within which a file may continue in the neighbouring range. */
 const EDGE_ARTICLES = 50_000
@@ -112,8 +112,8 @@ class SeedService {
     await SfvService.claimAll(added)
     const candidates = added
       .filter((file) => ContentService.isCandidate(file.name))
-      .map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
-    if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates)
+      .map((file) => file.id)
+    if (candidates.length > 0) await InspectionQueue.add(candidates)
     result.inspections = candidates.length
     return result
   }

commands/ContentBackfill.ts

@@ -4,6 +4,7 @@ import db from '@adonisjs/lucid/services/db'
 import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
 import { isArchiveStart } from '#lib/content/index'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
 
 const BATCH = 1000
 
@@ -71,19 +72,15 @@ export default class ContentBackfill extends BaseCommand {
         const jobs = files
           .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
           .slice(0, this.limit ? this.limit - queued : undefined)
-          .map((f) => ({
-            name: 'process-collection',
-            data: { fileId: Number(f.id) },
-            // A new id per run: BullMQ ignores an add() for an id it still knows.
-            // Targeted backfills go ahead of whatever is waiting.
-            opts: {
-              jobId: `content-${f.id}-${Date.now()}`,
-              lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
-              // niconico captures are many (~190k) and only cosmetic: behind everything else.
-              ...(this.nicovideo ? { priority: 10 } : {}),
-            },
-          }))
-        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+          .map((f) => Number(f.id))
+        // Into the inspection backlog (InspectionQueue): targeted backfills go ahead of what's waiting;
+        // niconico captures are many (~190k) and only cosmetic, so behind everything else.
+        const priority = this.nicovideo
+          ? INSPECTION_PRIORITY.low
+          : this.archiveNzbs || this.sevenZip || this.archives || this.sfv
+            ? INSPECTION_PRIORITY.targeted
+            : INSPECTION_PRIORITY.normal
+        if (jobs.length > 0) await InspectionQueue.add(jobs, priority)
         queued += jobs.length
       }
       this.logger.info(`Queued ${queued} files for content inspection.`)

commands/IndexScheduler.ts

@@ -5,6 +5,7 @@ import NntpService from '#services/NntpService'
 import QueueService from '#services/QueueService'
 import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
+import InspectionQueue from '#services/InspectionQueue'
 import FetchRangeService from '#services/FetchRangeService'
 import SplitService from '#services/SplitService'
 import BackfillService from '#services/BackfillService'
@@ -108,6 +109,12 @@ export default class IndexScheduler extends BaseCommand {
       }
     }
 
+    // The inspection backlog lives in Postgres; BullMQ gets a working set (LOG.md 2026-10-07 (8)).
+    const feedInspection = async () => {
+      const moved = await InspectionQueue.feed()
+      if (moved > 0) this.logger.info(`Inspection: ${moved} files moved from the backlog to the queue.`)
+    }
+
     const backfill = async () => {
       try {
         const queued = await BackfillService.tick()
@@ -122,7 +129,7 @@ export default class IndexScheduler extends BaseCommand {
     // here once ended the loop while the process stayed up, and live indexing stopped for ~3 hours
     // (LOG.md 2026-10-07 (3)).
     while (!Shutdown.requested) {
-      for (const step of [schedule, backfill, split]) {
+      for (const step of [schedule, backfill, split, feedInspection]) {
         try {
           await step()
         } catch (err: any) {

commands/InspectionFromRedis.ts

@@ -0,0 +1,56 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import QueueService from '#services/QueueService'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
+
+/**
+ * One-off (LOG.md 2026-10-07 (8)): moves the inspection jobs waiting in
+ * BullMQ into the inspection_queue table, then empties BullMQ's waiting and
+ * prioritized jobs so Redis gives the memory back. The file id is read from
+ * the job id (file-<id>, content-<id>-<time>), so no job is loaded.
+ * index-scheduler then feeds a working set back in.
+ */
+export default class InspectionFromRedis extends BaseCommand {
+  public static commandName = 'inspection:from-redis'
+  public static description = 'Moves waiting inspection jobs from Redis into the inspection_queue table.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count, change nothing' })
+  declare dryRun: boolean
+
+  public async run() {
+    const queue = QueueService.collectionQueue
+    const client = await queue.client
+    const prefix = `${queue.opts.prefix ?? 'bull'}:${queue.name}`
+    try {
+      const idOf = (jobId: string) => {
+        const m = /^(?:file|content)-(\d+)/.exec(jobId)
+        return m ? Number(m[1]) : null
+      }
+      let moved = 0
+      let skipped = 0
+      const take = async (ids: string[], priority: number) => {
+        const fileIds = ids.map(idOf).filter((id): id is number => id !== null)
+        skipped += ids.length - fileIds.length
+        if (!this.dryRun) await InspectionQueue.add(fileIds, priority)
+        moved += fileIds.length
+      }
+      const waiting = await client.llen(`${prefix}:wait`)
+      for (let i = 0; i < waiting; i += 50_000) await take(await client.lrange(`${prefix}:wait`, i, i + 49_999), INSPECTION_PRIORITY.normal)
+      // Prioritized jobs were the low-priority niconico captures.
+      const prioritized = await client.zcard(`${prefix}:prioritized`)
+      for (let i = 0; i < prioritized; i += 50_000) await take(await client.zrange(`${prefix}:prioritized`, i, i + 49_999), INSPECTION_PRIORITY.low)
+      this.logger.info(`${this.dryRun ? 'Would move' : 'Moved'} ${moved} jobs into inspection_queue (${skipped} with other ids skipped).`)
+      if (this.dryRun) return
+      // Waiting and prioritized jobs only; active ones finish as they are.
+      await queue.drain()
+      const counts = await queue.getJobCounts('wait', 'prioritized', 'active')
+      this.logger.info(`BullMQ now: ${counts.wait} waiting, ${counts.prioritized} prioritized, ${counts.active} active; table: ${await InspectionQueue.size()}.`)
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+}

database/migrations/1768620765260_inspection_queue.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * The content inspection backlog (LOG.md 2026-10-07 (8)): files waiting to be
+ * inspected, in Postgres instead of as BullMQ jobs. 5.4M waiting jobs took
+ * 3.3 GB of Redis memory and most of its single thread; InspectionQueue.feed
+ * keeps at most INSPECTION_FEED_MAX of them in BullMQ, where the workers and
+ * nodes take them. Lower priority first, then oldest.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE TABLE IF NOT EXISTS inspection_queue (
+        file_id bigint PRIMARY KEY,
+        priority smallint NOT NULL DEFAULT 0,
+        queued_at timestamptz NOT NULL DEFAULT now())`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS inspection_queue_order_idx ON inspection_queue (priority, queued_at)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE IF EXISTS inspection_queue`)
+  }
+}

database/schema.ts

@@ -371,6 +371,17 @@ export class GroupSchema extends BaseModel {
   declare updatedAt: DateTime | null
 }
 
+export class InspectionQueueSchema extends BaseModel {
+  static $columns = ['fileId', 'priority', 'queuedAt'] as const
+  $columns = InspectionQueueSchema.$columns
+  @column({ isPrimary: true })
+  declare fileId: bigint | number
+  @column()
+  declare priority: number
+  @column.dateTime()
+  declare queuedAt: DateTime
+}
+
 export class PartConflictSchema extends BaseModel {
   static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const
   $columns = PartConflictSchema.$columns

start/env.ts

@@ -107,6 +107,7 @@ export default await Env.create(new URL('../', import.meta.url), {
   FILE_KEYS_FILL_SECONDS: Env.schema.number.optional(),
   GC_AWAIT_INSPECTION_SECONDS: Env.schema.number.optional(),
   GC_MERGE_INTERVAL_MS: Env.schema.number.optional(),
+  INSPECTION_FEED_MAX: Env.schema.number.optional(),
   ARCHIVE_WALK_MAX_SEGMENTS: Env.schema.number.optional(),
   DEFERRED_SETTLE_MS: Env.schema.number.optional(),
   DEFERRED_EXPAND_MAX_PARTS: Env.schema.number.optional(),

tests/functional/inspection_queue.spec.ts

@@ -0,0 +1,36 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import QueueService from '#services/QueueService'
+import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
+
+test.group('Inspection queue', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.teardown(() => QueueService.closeAll())
+
+  test('the backlog stays in Postgres; BullMQ gets what fits, in priority order', async ({ assert }) => {
+    const queue = QueueService.collectionQueue
+    await queue.drain()
+    await InspectionQueue.add([9_000_001, 9_000_002], INSPECTION_PRIORITY.low)
+    await InspectionQueue.add([9_000_003])
+    await InspectionQueue.add([9_000_004], INSPECTION_PRIORITY.targeted)
+    // Queued again with a higher priority: one row, the higher priority.
+    await InspectionQueue.add([9_000_001], INSPECTION_PRIORITY.targeted)
+    assert.equal(await InspectionQueue.size(), 4)
+
+    const max = InspectionQueue.feedMax
+    ;(InspectionQueue as any).feedMax = 2
+    try {
+      assert.equal(await InspectionQueue.feed(), 2)
+      const waiting = (await queue.getWaiting()).map((job) => job.data.fileId).sort()
+      assert.deepEqual(waiting, [9_000_001, 9_000_004])
+      // Full: nothing more until the workers take some.
+      assert.equal(await InspectionQueue.feed(), 0)
+      const left = (await db.from('inspection_queue').select('file_id')).map((r) => Number(r.file_id)).sort()
+      assert.deepEqual(left, [9_000_002, 9_000_003])
+    } finally {
+      ;(InspectionQueue as any).feedMax = max
+      await queue.drain()
+    }
+  })
+})