69 lines
2.6 KiB
TypeScript
69 lines
2.6 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import QueueService from '#services/QueueService'
|
|
import ContentService from '#services/ContentService'
|
|
|
|
const BATCH = 1000
|
|
|
|
/**
|
|
* Queues completed files for content inspection (names and groupings from
|
|
* par2 and RAR headers, CONCEPT.md D21) that were completed before it
|
|
* existed. Costs one article download per file, done by worker:collection.
|
|
*/
|
|
export default class ContentBackfill extends BaseCommand {
|
|
public static commandName = 'content:backfill'
|
|
public static description = 'Queues existing files with random names for content inspection.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
@flags.number({ description: 'Queue at most this many files' })
|
|
declare limit?: number
|
|
|
|
@flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
|
|
declare all: boolean
|
|
|
|
@flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
|
|
declare archiveNzbs: boolean
|
|
|
|
public async run() {
|
|
let lastId = 0
|
|
let queued = 0
|
|
try {
|
|
while (!this.limit || queued < this.limit) {
|
|
// Files never inspected: no fingerprint on their binary and no archive listing.
|
|
const query = db
|
|
.from('files as f')
|
|
.join('binaries as b', 'b.file_id', 'f.id')
|
|
.where('f.id', '>', lastId)
|
|
if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
|
|
else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
|
|
const files = await query
|
|
.orderBy('f.id')
|
|
.limit(BATCH)
|
|
.select('f.id', 'f.filename')
|
|
if (files.length === 0) break
|
|
lastId = Number(files.at(-1).id)
|
|
|
|
const jobs = files
|
|
.filter((f) => ContentService.isCandidate(f.filename))
|
|
.slice(0, this.limit ? this.limit - queued : undefined)
|
|
.map((f) => ({
|
|
name: 'process-collection',
|
|
data: { fileId: Number(f.id) },
|
|
// A new id per run: BullMQ ignores an add() for an id it still knows.
|
|
// Targeted backfills go ahead of whatever is waiting.
|
|
opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
|
|
}))
|
|
if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
|
|
queued += jobs.length
|
|
}
|
|
this.logger.info(`Queued ${queued} files for content inspection.`)
|
|
} finally {
|
|
await QueueService.closeAll()
|
|
}
|
|
}
|
|
}
|