usenet-indexer/commands/releases_readable.ts

import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { isUnreadableName } from '#lib/release/index'

/** Its place in worker_cursors (the last release id done), for a run that was stopped. */
const CURSOR = 'readable'

/**
 * The readable flag of the catalog's releases by today's classifier (isUnreadableName, LOG.md (129)): rows flagged
 * readable whose name no one can read ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C") get readable =
 * false, so the readable filter hides them (an inner name from the prober still shows them), and their probe
 * candidates marked unreadable, the prober's first tiers. By the release name,
 * which is the shared file stem for most rows; not by the chunks' file names, which the edge judges from now on.
 * Batches by id, a dry run by default; --apply writes, and goes on from its cursor when run again.
 */
export default class ReleasesReadable extends BaseCommand {
  static commandName = 'releases:readable'
  static description =
    'Clears the readable flag of catalog rows whose name is unreadable by the current classifier; counts only without --apply'
  static options: CommandOptions = { startApp: true }

  @flags.number({ description: 'Release ids per batch' })
  declare batch: number

  @flags.number({ description: 'Stop after this id (a sample)' })
  declare toId: number

  @flags.boolean({ description: 'Write the flag; without it, count and show a sample' })
  declare apply: boolean

  @flags.boolean({ description: 'Start from the first id, not the cursor' })
  declare restart: boolean

  async run() {
    const batch = this.batch ?? 20_000
    const { rows: ends } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM releases`)
    const hi = Math.min(Number(ends[0].hi), this.toId ?? Infinity)
    const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [
      CURSOR,
    ])
    let after =
      this.restart || !this.apply || !cursors[0] ? Number(ends[0].lo) - 1 : Number(cursors[0].key)
    let looked = 0
    let changed = 0
    const sample: string[] = []
    const started = Date.now()
    while (after < hi) {
      const to = Math.min(after + batch, hi)
      const { rows } = await db.rawQuery(
        `SELECT id, name FROM releases WHERE id > ? AND id <= ? AND readable`,
        [after, to]
      )
      looked += rows.length
      const ids: number[] = []
      for (const r of rows) {
        if (!isUnreadableName(r.name)) continue
        ids.push(Number(r.id))
        if (sample.length < 40) sample.push(r.name)
      }
      changed += ids.length
      if (this.apply) {
        if (ids.length > 0) {
          await db.rawQuery(`UPDATE releases SET readable = false WHERE id = ANY(?::bigint[])`, [
            ids,
          ])
          // Their probe candidates to the unreadable tiers, which the prober takes first (LOG.md (130)).
          await db.rawQuery(
            `UPDATE probe_candidates SET unreadable = true WHERE release_id = ANY(?::bigint[]) AND NOT unreadable`,
            [ids]
          )
        }
        await db.rawQuery(
          `INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
          [CURSOR, String(to)]
        )
      }
      after = to
      if (Math.floor(after / 2_000_000) !== Math.floor((after - batch) / 2_000_000)) {
        this.logger.info(
          `Up to id ${after} of ${hi}: ${changed} unreadable of ${looked} readable (${Math.round((Date.now() - started) / 1000)} s).`
        )
      }
    }
    this.logger.info(
      `${this.apply ? 'Cleared' : 'Would clear'} ${changed} of ${looked} readable rows.`
    )
    if (!this.apply) for (const n of sample) this.logger.info(`  ${n}`)
  }
}