usenet-indexer/commands/releases_regroup.ts

import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import { decodeChunk } from '#lib/chunks/index'
import { releaseIdentity } from '#lib/release/index'
import { sourceCode } from '#lib/release/source'

/**
 * What lib/release's identity makes of releases already in the catalog: a
 * sample of releases (by source), their files decoded from the chunks and
 * each given its release key again. Shows how many would now split, and
 * into what: for checking an identity change on real data.
 */
export default class ReleasesRegroup extends BaseCommand {
  static commandName = 'releases:regroup'
  static description = 'Shows how the current release identity would regroup a sample of the catalog'
  static options: CommandOptions = { startApp: true }

  @flags.number({ description: 'Releases sampled per source', default: 2000 })
  declare sample: number

  @flags.array({ description: 'Release ids to show in full' })
  declare show?: string[]

  @flags.array({ description: 'Groups whose indexed releases are re-keyed together (merges show too)' })
  declare group?: string[]

  async run() {
    for (const name of this.group ?? []) {
      const { rows } = await db.rawQuery(
        `SELECT DISTINCT c.release_id FROM release_chunks c JOIN blocks b ON b.id = c.block_id JOIN groups g ON g.id = b.group_id WHERE g.name = ?`,
        [name]
      )
      const ids = rows.map((r: { release_id: string }) => r.release_id)
      const all = new Map<string, { name: string; source: string; files: number; subject?: string }>()
      for (const id of ids) {
        for (const [key, g] of await this.regroup(Number(id))) {
          const known = all.get(key)
          if (known) known.files += g.files
          else all.set(key, { ...g })
        }
      }
      const sizes = [...all.values()].map((g) => g.files)
      const single = sizes.filter((n) => n === 1).length
      this.logger.info(`${name}: ${ids.length} releases now, ${all.size} with the current rule (${single} of one file)`)
      for (const g of [...all.values()].sort((a, b) => b.files - a.files).slice(0, 6)) this.logger.info(`  ${g.files} files: ${g.name.slice(0, 120)} (${g.source})`)
      const singles = [...all.values()].filter((g) => g.files === 1)
      for (let i = 0; i < Math.min(8, singles.length); i++) this.logger.info(`  single: ${singles[Math.floor((i * singles.length) / 8)].subject?.slice(0, 150)}`)
    }
    if (this.group?.length) return
    for (const id of this.show ?? []) {
      const groups = await this.regroup(Number(id))
      this.logger.info(`${id}: ${groups.size} release(s) now`)
      for (const [, g] of groups) this.logger.info(`  ${g.files} files: ${g.name} (${g.source})`)
    }
    if (this.show?.length) return
    for (const source of ['subject', 'post', 'stem']) {
      const rows = await db.from('releases').where('source', sourceCode(source)).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
      let split = 0
      let pieces = 0
      const examples: string[] = []
      for (const r of rows) {
        const groups = await this.regroup(Number(r.id))
        if (groups.size <= 1) continue
        split++
        pieces += groups.size
        if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`)
      }
      this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`)
      for (const e of examples) this.logger.info(`  ${e.slice(0, 260)}`)
    }
  }

  private async regroup(id: number) {
    const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
    const groups = new Map<string, { name: string; source: string; files: number; subject?: string }>()
    for (const c of chunks) {
      for (const f of decodeChunk(c.payload)) {
        const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
        const g = groups.get(r.key)
        if (g) g.files++
        else groups.set(r.key, { name: r.name, source: r.source, files: 1, subject: f.subject })
      }
    }
    return groups
  }
}