usenet-indexer/commands/SeedRange.ts

import { BaseCommand, args, flags } from '@adonisjs/core/ace'
import { CommandOptions } from '@adonisjs/core/types/ace'
import nntpConfig from '#config/nntp'
import SeedService from '#services/SeedService'
import QueueService from '#services/QueueService'
import { isDeferred } from '#services/DeferredService'
import { XoverClient } from '#lib/nntp/xover_client'
import { runRange } from '#lib/seed/runner'
import type { SeedBatch } from '#lib/seed/wire'

/**
 * Edge aggregation (CONCEPT.md D37) on this machine: indexes a range of a
 * group's history over one NNTP connection (gzip-compressed overviews when
 * the server has them), grouping headers into files in memory and storing
 * one row per file directly. seed:node does the same for a remote node,
 * through the seed API. Header-only, like the backfill.
 *
 *   node ace seed:range alt.binaries.ath --start 46000000000 --count 2000000
 *   node ace seed:range alt.binaries.ath --start 46000000000 --count 2000000 --dry-run
 */
export default class SeedRange extends BaseCommand {
  public static commandName = 'seed:range'
  public static description = "Indexes a range of a group's headers by aggregating them into files on this node."

  public static options: CommandOptions = {
    startApp: true,
  }

  @args.string({ description: 'Group' })
  declare group: string

  @flags.string({ description: 'First article number' })
  declare start: string

  @flags.number({ description: 'Articles to index', default: 1_000_000 })
  declare count: number

  @flags.number({ description: 'Articles per XOVER', default: 20_000 })
  declare chunk: number

  @flags.number({ description: 'Batches without a new part before an open file is given up as fragments', default: 5 })
  declare idle: number

  @flags.boolean({ description: 'Fetch and aggregate only; store nothing' })
  declare dryRun: boolean

  @flags.boolean({ description: 'Ask the server for gzip-compressed overviews', default: true })
  declare compress: boolean

  public async run() {
    const first = Number(this.start)
    if (!Number.isSafeInteger(first) || first < 1) throw new Error(`--start must be an article number, got ${this.start}`)
    const last = first + this.count - 1
    const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, fragments: 0, inspections: 0 }
    let storeMs = 0
    const cpuStart = process.cpuUsage()
    const started = Date.now()

    const client = await XoverClient.connect({
      host: nntpConfig.host,
      port: nntpConfig.port,
      secure: nntpConfig.secure,
      servername: nntpConfig.tlsServername,
      user: nntpConfig.user,
      password: nntpConfig.password?.release(),
      timeoutMs: nntpConfig.commandTimeoutMs,
      compress: this.compress,
    })
    this.logger.info(`Connected${client.compressed ? ', overviews gzip-compressed' : ''}.`)
    try {
      await runRange({
        client,
        group: this.group,
        start: first,
        end: last,
        chunk: this.chunk,
        idle: this.idle,
        submitEvery: 1,
        isDeferred,
        sink: async (batch: SeedBatch) => {
          stats.headers += batch.headers
          stats.obfuscated += batch.obfuscated
          stats.deferred += batch.deferred
          stats.complete += batch.files.length
          stats.fragments += batch.fragments.length
          if (!this.dryRun) {
            const t = Date.now()
            const stored = await SeedService.store(batch.files)
            await SeedService.storeFragments(batch.fragments, { start: first, end: last })
            storeMs += Date.now() - t
            stats.files += stored.files
            stats.existing += stored.existing
            stats.inspections += stored.inspections
          }
          this.progress(stats, started)
        },
      })
    } finally {
      client.close()
      await QueueService.closeAll()
    }

    const seconds = (Date.now() - started) / 1000
    const cpu = process.cpuUsage(cpuStart)
    const n = (x: number) => x.toLocaleString('en')
    this.logger.info(
      `${this.group} ${first}-${last}: ${n(stats.headers)} headers in ${seconds.toFixed(0)} s (${n(Math.round(stats.headers / seconds))}/s). ` +
        `${n(stats.obfuscated)} obfuscated, ${n(stats.deferred)} deferred. ` +
        `${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known), ` +
        `${n(stats.fragments)} fragment segments, ${n(stats.inspections)} queued for inspection.`
    )
    this.logger.info(`Store ${(storeMs / 1000).toFixed(0)} s; node CPU ${((cpu.user + cpu.system) / 1e6).toFixed(0)} s.`)
  }

  private lastReport = 0
  private progress(stats: Record<string, number>, started: number) {
    if (Date.now() - this.lastReport < 30_000) return
    this.lastReport = Date.now()
    const seconds = (Date.now() - started) / 1000
    this.logger.info(
      `${stats.headers.toLocaleString('en')} headers (${Math.round(stats.headers / seconds).toLocaleString('en')}/s), ` +
        `${stats.complete.toLocaleString('en')} files, ${stats.fragments.toLocaleString('en')} fragment segments.`
    )
  }
}