implemented a lot of features

commands/ContentBackfill.ts

@@ -0,0 +1,68 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import QueueService from '#services/QueueService'
+import ContentService from '#services/ContentService'
+
+const BATCH = 1000
+
+/**
+ * Queues completed files for content inspection (names and groupings from
+ * par2 and RAR headers, CONCEPT.md D21) that were completed before it
+ * existed. Costs one article download per file, done by worker:collection.
+ */
+export default class ContentBackfill extends BaseCommand {
+  public static commandName = 'content:backfill'
+  public static description = 'Queues existing files with random names for content inspection.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Queue at most this many files' })
+  declare limit?: number
+
+  @flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
+  declare all: boolean
+
+  @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
+  declare archiveNzbs: boolean
+
+  public async run() {
+    let lastId = 0
+    let queued = 0
+    try {
+      while (!this.limit || queued < this.limit) {
+        // Files never inspected: no fingerprint on their binary and no archive listing.
+        const query = db
+          .from('files as f')
+          .join('binaries as b', 'b.file_id', 'f.id')
+          .where('f.id', '>', lastId)
+        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
+        else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
+        const files = await query
+          .orderBy('f.id')
+          .limit(BATCH)
+          .select('f.id', 'f.filename')
+        if (files.length === 0) break
+        lastId = Number(files.at(-1).id)
+
+        const jobs = files
+          .filter((f) => ContentService.isCandidate(f.filename))
+          .slice(0, this.limit ? this.limit - queued : undefined)
+          .map((f) => ({
+            name: 'process-collection',
+            data: { fileId: Number(f.id) },
+            // A new id per run: BullMQ ignores an add() for an id it still knows.
+            // Targeted backfills go ahead of whatever is waiting.
+            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
+          }))
+        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+        queued += jobs.length
+      }
+      this.logger.info(`Queued ${queued} files for content inspection.`)
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+}