import { BaseCommand, flags } from '@adonisjs/core/ace' import type { CommandOptions } from '@adonisjs/core/types/ace' import db from '@adonisjs/lucid/services/db' import { isUnreadableName } from '#lib/release/index' /** Its place in worker_cursors (the last release id done), for a run that was stopped. */ const CURSOR = 'readable' /** * The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged * readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable = * false, so the readable filter hides them (an inner name from the prober still shows them), and their probe * candidates marked unreadable, the prober's first tiers. By the release name, * which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on. * Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again. */ export default class ReleasesReadable extends BaseCommand { static commandName = 'releases:readable' static description = 'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply' static options: CommandOptions = { startApp: true } @flags.number({ description: 'Release ids per batch' }) declare batch: number @flags.number({ description: 'Stop after this id (a sample)' }) declare toId: number @flags.boolean({ description: 'Write the flag; without it, count and show a sample' }) declare apply: boolean @flags.boolean({ description: 'Start from the first id, not the cursor' }) declare restart: boolean async run() { const batch = this.batch ?? 20_000 const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`) const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity) const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [ CURSOR, ]) let after = this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key) let looked = 0 let changed = 0 const sample: string[] = [] const started = Date.now() while (after < hi) { const to = Math.min(after + batch, hi) const { rows } = await db.rawQuery( `SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`, [after, to] ) looked += rows.length const ids: number[] = [] for (const r of rows) { if (!isUnreadableName(r.name)) continue ids.push(Number(r.id)) if (sample.length < 40) sample.push(r.name) } changed += ids.length if (this.apply) { if (ids.length > 0) { await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [ ids, ]) // Their probe candidates to the unreadable tiers, which the prober takes first (LOG.md (130)). await db.rawQuery( `UPDATE probe_candidates SET unreadable = true WHERE release_id = ANY(?::bigint[]) AND NOT unreadable`, [ids] ) } await db.rawQuery( `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`, [CURSOR, String(to)] ) } after = to if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) { this.logger.info( `Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).` ) } } this.logger.info( `${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.` ) if (!this.apply) for (const n of sample) this.logger.info(` ${n}`) } }