usenet-indexer/commands/GroupsTop.ts

import { writeFile } from 'node:fs/promises'
import { setTimeout as sleep } from 'node:timers/promises'
import { BaseCommand, flags } from '@adonisjs/core/ace'
import { CommandOptions } from '@adonisjs/core/types/ace'
import NntpService from '#services/NntpService'
import indexerConfig from '#config/indexer'
import { parseGroupResponse } from '#lib/nntp/group'
import { parseSubject } from '#lib/subject/index'

interface GroupRate {
  name: string
  perMinute: number
  sampled?: number
  normal?: number
  obfuscated?: number
  skipped?: number
  avgArticleBytes?: number
  avgHeaderBytes?: number
}

/**
 * Lists the busiest newsgroups (by articles posted per minute, from two
 * LIST ACTIVE snapshots) and optionally samples their newest headers to
 * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)).
 *
 *   node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt
 */
export default class GroupsTop extends BaseCommand {
  public static commandName = 'groups:top'
  public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.'

  public static options: CommandOptions = {
    startApp: true,
  }

  @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' })
  declare pattern: string

  @flags.number({ description: 'Minutes between the two snapshots', default: 5 })
  declare minutes: number

  @flags.number({ description: 'Number of groups to list', default: 1000 })
  declare limit: number

  @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 })
  declare sample: number

  @flags.string({ description: 'Write the list to this file (default: print only)' })
  declare out?: string

  public async run() {
    const conn = await NntpService.acquire()
    let failed = false
    try {
      this.logger.info(`Snapshot 1 of ${this.pattern}…`)
      const first = await this.snapshot(conn)
      this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`)
      const started = Date.now()
      await sleep(this.minutes * 60_000)
      const second = await this.snapshot(conn)
      const minutes = (Date.now() - started) / 60_000

      const groups: GroupRate[] = []
      for (const [name, high] of second) {
        const before = first.get(name)
        if (before === undefined || high <= before) continue
        groups.push({ name, perMinute: Number(high - before) / minutes })
      }
      groups.sort((a, b) => b.perMinute - a.perMinute)
      const top = groups.slice(0, this.limit)
      this.logger.info(`${groups.length} active groups; listing ${top.length}.`)

      if (this.sample > 0) {
        for (const [i, group] of top.entries()) {
          try {
            Object.assign(group, await this.sampleGroup(conn, group.name))
          } catch (error: any) {
            this.logger.warning(`${group.name}: ${error.message}`)
          }
          if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`)
        }
      }

      const text = this.format(top, minutes)
      if (this.out) {
        await writeFile(this.out, text)
        this.logger.info(`Written to ${this.out}`)
      } else {
        console.log(text)
      }
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
      await NntpService.shutdown()
    }
  }

  /** Group name → highest article number, from LIST ACTIVE. */
  private async snapshot(conn: any): Promise<Map<string, bigint>> {
    const { data } = await conn.list(this.pattern)
    const result = new Map<string, bigint>()
    for (const line of Buffer.from(data).toString('latin1').split('\r\n')) {
      const [name, high] = line.split(' ')
      if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high))
    }
    return result
  }

  /** Classifies the newest headers the way worker:fetch does. */
  private async sampleGroup(conn: any, name: string) {
    const { last } = parseGroupResponse((await conn.group(name)).resp)
    const start = last - BigInt(this.sample) + 1n
    const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString())
    let normal = 0
    let skipped = 0
    let obfuscated = 0
    let obfuscatedBytes = 0
    let headerBytes = 0
    for (const [, header] of overviews as [number, Record<string, string>][]) {
      if (!header) continue
      headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0)
      if (header.subject && parseSubject(header.subject)) normal++
      else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
      else {
        obfuscated++
        obfuscatedBytes += Number(header[':bytes']) || 0
      }
    }
    const sampled = normal + skipped + obfuscated
    return {
      sampled,
      normal,
      skipped,
      obfuscated,
      avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0,
      avgHeaderBytes: sampled ? headerBytes / sampled : 0,
    }
  }

  private format(groups: GroupRate[], minutes: number): string {
    const sampled = groups.some((g) => g.sampled)
    const lines = [
      `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`,
      sampled
        ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s'
        : '# rank\tgroup\tarticles/min',
    ]
    let bodyTotal = 0
    let headerTotal = 0
    let articlesTotal = 0
    for (const [i, g] of groups.entries()) {
      articlesTotal += g.perMinute
      if (!g.sampled) {
        lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`)
        continue
      }
      const obfuscatedShare = g.obfuscated! / g.sampled
      // Bodies: every obfuscated article is downloaded once. Headers: every article.
      const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6
      const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6
      bodyTotal += bodyMbit
      headerTotal += headerMbit
      lines.push(
        [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t')
      )
    }
    lines.push(`# total: ${Math.round(articlesTotal)} articles/min`)
    if (sampled) {
      lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`)
      lines.push(`#   of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`)
    }
    return lines.join('\n') + '\n'
  }
}