usenet-indexer/tools/obf_runs.ts

/**
 * How obfuscated subjects (no quoted file name or no (i/n) counter: the edge
 * worker only counts them) come in runs, on a captured range: whether a run
 * of articles can be tracked by article numbers instead of message-ids, and
 * how many runs there would be (LOG.md 2026-10-11 (5)).
 *
 *   node --import=@poppinss/ts-exec tools/obf_runs.ts <capture dir> [gapMinutes]
 *
 * Strict runs: consecutive obfuscated articles (in article-number order) of
 * one poster and posting tool. Keyed runs: per (poster, tool), open while
 * articles come within gapMinutes of each other, other posts in between.
 * Writes <capture>/obf-runs.tsv (keyed runs, for the body sample).
 */
import { readFile, readdir, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { unzipSync } from 'node:zlib'
import { SegmentScanner, scanOverviewText } from '#lib/edge/scan'
import { uploadTool } from '#lib/rules/index'

const [dir, gapArg] = process.argv.slice(2)
const gapMs = Number(gapArg ?? 10) * 60_000
const { group } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
const scanner = new SegmentScanner()
const priorities = { [group]: 1 }

interface Run { key: string; poster: string; tool: string; first: number; last: number; articles: number; bytes: number; sizes: Map<number, number>; firstAt: number; lastAt: number; ids: number[] }
let lines = 0, owned = 0, obfuscated = 0
let strict = null as Run | null
const strictRuns: number[] = []
const open = new Map<string, Run>()
const keyed: Run[] = []
const posters = new Set<string>()

const close = (r: Run) => keyed.push(r)
for (const file of (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()) {
  const text = unzipSync(await readFile(join(dir, file))).toString('utf8')
  scanOverviewText(text, (n, subject, from, date, messageId, bytes, xref) => {
    lines++
    if (xref && scanner.owner(xref, group, priorities) !== group) return
    owned++
    if (scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) {
      if (strict) strictRuns.push(strict.articles), (strict = null)
      return
    }
    obfuscated++
    const tool = uploadTool(messageId)
    const key = `${from}\t${tool}`
    posters.add(from)
    const at = Date.parse(date.replace(/ UTC$/, ' GMT')) || 0
    const size = Number(bytes) || 0
    if (!strict || strict.key !== key) {
      if (strict) strictRuns.push(strict.articles)
      strict = { key, poster: from, tool, first: n, last: n, articles: 0, bytes: 0, sizes: new Map(), firstAt: at, lastAt: at, ids: [] }
    }
    strict.articles++
    let run = open.get(key)
    if (run && at - run.lastAt > gapMs) {
      close(run)
      run = undefined
    }
    if (!run) {
      run = { key, poster: from, tool, first: n, last: n, articles: 0, bytes: 0, sizes: new Map(), firstAt: at, lastAt: at, ids: [] }
      open.set(key, run)
    }
    run.last = n
    run.articles++
    run.bytes += size
    run.sizes.set(size, (run.sizes.get(size) ?? 0) + 1)
    run.lastAt = Math.max(run.lastAt, at)
    if (run.ids.length < 3) run.ids.push(n)
  })
}
if (strict !== null) strictRuns.push((strict as Run).articles)
for (const r of open.values()) close(r)

const pct = (a: number, b: number) => `${((100 * a) / Math.max(1, b)).toFixed(1)}%`
const quant = (xs: number[], q: number) => xs.slice().sort((a, b) => a - b)[Math.floor(q * (xs.length - 1))] ?? 0
const keyedSizes = keyed.map((r) => r.articles)
console.log(`${dir}: ${lines} lines, ${owned} owned, ${obfuscated} obfuscated (${pct(obfuscated, owned)} of owned), ${posters.size} distinct posters among them`)
console.log(`strict runs: ${strictRuns.length} (${(obfuscated / Math.max(1, strictRuns.length)).toFixed(1)} articles a run; median ${quant(strictRuns, 0.5)}, p90 ${quant(strictRuns, 0.9)})`)
console.log(`keyed runs (gap ${gapMs / 60000} min): ${keyed.length} (${(obfuscated / Math.max(1, keyed.length)).toFixed(1)} articles a run; median ${quant(keyedSizes, 0.5)}, p90 ${quant(keyedSizes, 0.9)}, max ${quant(keyedSizes, 1)})`)
const span = keyed.map((r) => (r.last - r.first + 1) / r.articles)
console.log(`keyed run density (article-number span per article): median ${quant(span, 0.5).toFixed(1)}, p90 ${quant(span, 0.9).toFixed(1)}`)
// A file posted in equal parts: one size for most articles of a run.
const modal = keyed.filter((r) => r.articles >= 10).map((r) => Math.max(...r.sizes.values()) / r.articles)
console.log(`runs of 10+ articles: ${modal.length}; share of their articles at the run's most common size: median ${pct(quant(modal, 0.5), 1)}, p10 ${pct(quant(modal, 0.1), 1)}`)
const byTool = new Map<string, number>()
for (const r of keyed) byTool.set(r.tool, (byTool.get(r.tool) ?? 0) + r.articles)
console.log('obfuscated articles by tool:', [...byTool].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([t, n]) => `${t} ${pct(n, obfuscated)}`).join(', '))
await writeFile(join(dir, 'obf-runs.tsv'), keyed.sort((a, b) => b.articles - a.articles).map((r) => [r.first, r.last, r.articles, r.bytes, r.tool, r.ids.join(','), r.poster].join('\t')).join('\n') + '\n')