implemented a lot of features

commands/GroupsTop.ts

@@ -0,0 +1,176 @@
+import { writeFile } from 'node:fs/promises'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import NntpService from '#services/NntpService'
+import indexerConfig from '#config/indexer'
+import { parseGroupResponse } from '#lib/nntp/group'
+import { parseSubject } from '#lib/subject/index'
+
+interface GroupRate {
+  name: string
+  perMinute: number
+  sampled?: number
+  normal?: number
+  obfuscated?: number
+  skipped?: number
+  avgArticleBytes?: number
+  avgHeaderBytes?: number
+}
+
+/**
+ * Lists the busiest newsgroups (by articles posted per minute, from two
+ * LIST ACTIVE snapshots) and optionally samples their newest headers to
+ * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)).
+ *
+ *   node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt
+ */
+export default class GroupsTop extends BaseCommand {
+  public static commandName = 'groups:top'
+  public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' })
+  declare pattern: string
+
+  @flags.number({ description: 'Minutes between the two snapshots', default: 5 })
+  declare minutes: number
+
+  @flags.number({ description: 'Number of groups to list', default: 1000 })
+  declare limit: number
+
+  @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 })
+  declare sample: number
+
+  @flags.string({ description: 'Write the list to this file (default: print only)' })
+  declare out?: string
+
+  public async run() {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      this.logger.info(`Snapshot 1 of ${this.pattern}…`)
+      const first = await this.snapshot(conn)
+      this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`)
+      const started = Date.now()
+      await sleep(this.minutes * 60_000)
+      const second = await this.snapshot(conn)
+      const minutes = (Date.now() - started) / 60_000
+
+      const groups: GroupRate[] = []
+      for (const [name, high] of second) {
+        const before = first.get(name)
+        if (before === undefined || high <= before) continue
+        groups.push({ name, perMinute: Number(high - before) / minutes })
+      }
+      groups.sort((a, b) => b.perMinute - a.perMinute)
+      const top = groups.slice(0, this.limit)
+      this.logger.info(`${groups.length} active groups; listing ${top.length}.`)
+
+      if (this.sample > 0) {
+        for (const [i, group] of top.entries()) {
+          try {
+            Object.assign(group, await this.sampleGroup(conn, group.name))
+          } catch (error: any) {
+            this.logger.warning(`${group.name}: ${error.message}`)
+          }
+          if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`)
+        }
+      }
+
+      const text = this.format(top, minutes)
+      if (this.out) {
+        await writeFile(this.out, text)
+        this.logger.info(`Written to ${this.out}`)
+      } else {
+        console.log(text)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+      await NntpService.shutdown()
+    }
+  }
+
+  /** Group name → highest article number, from LIST ACTIVE. */
+  private async snapshot(conn: any): Promise<Map<string, bigint>> {
+    const { data } = await conn.list(this.pattern)
+    const result = new Map<string, bigint>()
+    for (const line of Buffer.from(data).toString('latin1').split('\r\n')) {
+      const [name, high] = line.split(' ')
+      if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high))
+    }
+    return result
+  }
+
+  /** Classifies the newest headers the way worker:fetch does. */
+  private async sampleGroup(conn: any, name: string) {
+    const { last } = parseGroupResponse((await conn.group(name)).resp)
+    const start = last - BigInt(this.sample) + 1n
+    const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString())
+    let normal = 0
+    let skipped = 0
+    let obfuscated = 0
+    let obfuscatedBytes = 0
+    let headerBytes = 0
+    for (const [, header] of overviews as [number, Record<string, string>][]) {
+      if (!header) continue
+      headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0)
+      if (header.subject && parseSubject(header.subject)) normal++
+      else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
+      else {
+        obfuscated++
+        obfuscatedBytes += Number(header[':bytes']) || 0
+      }
+    }
+    const sampled = normal + skipped + obfuscated
+    return {
+      sampled,
+      normal,
+      skipped,
+      obfuscated,
+      avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0,
+      avgHeaderBytes: sampled ? headerBytes / sampled : 0,
+    }
+  }
+
+  private format(groups: GroupRate[], minutes: number): string {
+    const sampled = groups.some((g) => g.sampled)
+    const lines = [
+      `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`,
+      sampled
+        ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s'
+        : '# rank\tgroup\tarticles/min',
+    ]
+    let bodyTotal = 0
+    let headerTotal = 0
+    let articlesTotal = 0
+    for (const [i, g] of groups.entries()) {
+      articlesTotal += g.perMinute
+      if (!g.sampled) {
+        lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`)
+        continue
+      }
+      const obfuscatedShare = g.obfuscated! / g.sampled
+      // Bodies: every obfuscated article is downloaded once. Headers: every article.
+      const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6
+      const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6
+      bodyTotal += bodyMbit
+      headerTotal += headerMbit
+      lines.push(
+        [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t')
+      )
+    }
+    lines.push(`# total: ${Math.round(articlesTotal)} articles/min`)
+    if (sampled) {
+      lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`)
+      lines.push(`#   of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`)
+    }
+    return lines.join('\n') + '\n'
+  }
+}