import { writeFile } from 'node:fs/promises' import { setTimeout as sleep } from 'node:timers/promises' import { BaseCommand, flags } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import NntpService from '#services/NntpService' import indexerConfig from '#config/indexer' import { parseGroupResponse } from '#lib/nntp/group' import { parseSubject } from '#lib/subject/index' interface GroupRate { name: string perMinute: number sampled?: number normal?: number obfuscated?: number skipped?: number avgArticleBytes?: number avgHeaderBytes?: number } /** * Lists the busiest newsgroups (by articles posted per minute, from two * LIST ACTIVE snapshots) and optionally samples their newest headers to * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)). * * node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt */ export default class GroupsTop extends BaseCommand { public static commandName = 'groups:top' public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.' public static options: CommandOptions = { startApp: true, } @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' }) declare pattern: string @flags.number({ description: 'Minutes between the two snapshots', default: 5 }) declare minutes: number @flags.number({ description: 'Number of groups to list', default: 1000 }) declare limit: number @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 }) declare sample: number @flags.string({ description: 'Write the list to this file (default: print only)' }) declare out?: string public async run() { const conn = await NntpService.acquire() let failed = false try { this.logger.info(`Snapshot 1 of ${this.pattern}…`) const first = await this.snapshot(conn) this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`) const started = Date.now() await sleep(this.minutes * 60_000) const second = await this.snapshot(conn) const minutes = (Date.now() - started) / 60_000 const groups: GroupRate[] = [] for (const [name, high] of second) { const before = first.get(name) if (before === undefined || high <= before) continue groups.push({ name, perMinute: Number(high - before) / minutes }) } groups.sort((a, b) => b.perMinute - a.perMinute) const top = groups.slice(0, this.limit) this.logger.info(`${groups.length} active groups; listing ${top.length}.`) if (this.sample > 0) { for (const [i, group] of top.entries()) { try { Object.assign(group, await this.sampleGroup(conn, group.name)) } catch (error: any) { this.logger.warning(`${group.name}: ${error.message}`) } if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`) } } const text = this.format(top, minutes) if (this.out) { await writeFile(this.out, text) this.logger.info(`Written to ${this.out}`) } else { console.log(text) } } catch (error) { failed = true throw error } finally { NntpService.release(conn, failed) await NntpService.shutdown() } } /** Group name → highest article number, from LIST ACTIVE. */ private async snapshot(conn: any): Promise> { const { data } = await conn.list(this.pattern) const result = new Map() for (const line of Buffer.from(data).toString('latin1').split('\r\n')) { const [name, high] = line.split(' ') if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high)) } return result } /** Classifies the newest headers the way worker:fetch does. */ private async sampleGroup(conn: any, name: string) { const { last } = parseGroupResponse((await conn.group(name)).resp) const start = last - BigInt(this.sample) + 1n const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString()) let normal = 0 let skipped = 0 let obfuscated = 0 let obfuscatedBytes = 0 let headerBytes = 0 for (const [, header] of overviews as [number, Record][]) { if (!header) continue headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0) if (header.subject && parseSubject(header.subject)) normal++ else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++ else { obfuscated++ obfuscatedBytes += Number(header[':bytes']) || 0 } } const sampled = normal + skipped + obfuscated return { sampled, normal, skipped, obfuscated, avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0, avgHeaderBytes: sampled ? headerBytes / sampled : 0, } } private format(groups: GroupRate[], minutes: number): string { const sampled = groups.some((g) => g.sampled) const lines = [ `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`, sampled ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s' : '# rank\tgroup\tarticles/min', ] let bodyTotal = 0 let headerTotal = 0 let articlesTotal = 0 for (const [i, g] of groups.entries()) { articlesTotal += g.perMinute if (!g.sampled) { lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`) continue } const obfuscatedShare = g.obfuscated! / g.sampled // Bodies: every obfuscated article is downloaded once. Headers: every article. const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6 const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6 bodyTotal += bodyMbit headerTotal += headerMbit lines.push( [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t') ) } lines.push(`# total: ${Math.round(articlesTotal)} articles/min`) if (sampled) { lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`) lines.push(`# of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`) } return lines.join('\n') + '\n' } }