import { BaseCommand, flags } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import db from '@adonisjs/lucid/services/db' import QueueService from '#services/QueueService' import ContentService from '#services/ContentService' const BATCH = 1000 /** * Queues completed files for content inspection (names and groupings from * par2 and RAR headers, CONCEPT.md D21) that were completed before it * existed. Costs one article download per file, done by worker:collection. */ export default class ContentBackfill extends BaseCommand { public static commandName = 'content:backfill' public static description = 'Queues existing files with random names for content inspection.' public static options: CommandOptions = { startApp: true, } @flags.number({ description: 'Queue at most this many files' }) declare limit?: number @flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' }) declare all: boolean @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' }) declare archiveNzbs: boolean public async run() { let lastId = 0 let queued = 0 try { while (!this.limit || queued < this.limit) { // Files never inspected: no fingerprint on their binary and no archive listing. const query = db .from('files as f') .join('binaries as b', 'b.file_id', 'f.id') .where('f.id', '>', lastId) if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`) else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive') const files = await query .orderBy('f.id') .limit(BATCH) .select('f.id', 'f.filename') if (files.length === 0) break lastId = Number(files.at(-1).id) const jobs = files .filter((f) => ContentService.isCandidate(f.filename)) .slice(0, this.limit ? this.limit - queued : undefined) .map((f) => ({ name: 'process-collection', data: { fileId: Number(f.id) }, // A new id per run: BullMQ ignores an add() for an id it still knows. // Targeted backfills go ahead of whatever is waiting. opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs }, })) if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs) queued += jobs.length } this.logger.info(`Queued ${queued} files for content inspection.`) } finally { await QueueService.closeAll() } } }