import { BaseCommand, args, flags } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import db from '@adonisjs/lucid/services/db' import QueueService from '#services/QueueService' import NntpService from '#services/NntpService' import ArchiveWalkService from '#services/ArchiveWalkService' const BATCH = 5000 /** * Complete archive listings for releases inspected before archive walks * existed (CONCEPT.md D40): queues those whose listing leaves most of the * release unaccounted for. worker:collection walks them, one at a time and * behind inspection. With release ids, walks those right here instead. */ export default class ArchiveWalk extends BaseCommand { public static commandName = 'content:walk-archives' public static description = 'Queues releases with partial archive listings for a (low priority) header walk.' public static options: CommandOptions = { startApp: true, } @args.spread({ description: 'Release ids to walk now (instead of queueing)', required: false }) declare releaseIds?: string[] @flags.number({ description: 'Queue at most this many releases' }) declare limit?: number @flags.boolean({ description: 'Also releases that were walked before' }) declare again: boolean public async run() { try { if (this.releaseIds?.length) { for (const id of this.releaseIds) { this.logger.info(JSON.stringify(await ArchiveWalkService.walk(Number(id)))) } return } let lastId = 0 let queued = 0 while (!this.limit || queued < this.limit) { const query = db .from('releases as r') .where('r.id', '>', lastId) .whereNotNull('r.contents') .orderBy('r.id') .limit(BATCH) .select('r.id', 'r.size', 'r.contents_walked_at', db.raw(`( SELECT count(*) FROM jsonb_array_elements(r.contents)) AS n`), db.raw(`( SELECT coalesce(sum((e->>'size')::bigint), 0) FROM jsonb_array_elements(r.contents) e) AS listed`)) const rows = await query if (rows.length === 0) break lastId = Number(rows.at(-1).id) const jobs = rows .filter((r) => this.again || !r.contents_walked_at) .filter((r) => ArchiveWalkService.worthWalking(Number(r.n), Number(r.listed), Number(r.size))) .slice(0, this.limit ? this.limit - queued : undefined) .map((r) => ({ name: 'walk', data: { releaseId: Number(r.id) }, opts: { jobId: `walk-${r.id}-${Date.now()}` } })) if (jobs.length > 0) await QueueService.archiveWalkQueue.addBulk(jobs) queued += jobs.length } this.logger.info(`Queued ${queued} releases for an archive walk.`) } finally { await QueueService.closeAll() await NntpService.shutdown() } } }