tools: obf_runs and obf_kinds, measuring the obfuscated subjects

obf_runs: how obfuscated articles come in runs (strict, and per poster
and tool with a time gap). obf_kinds: what they are (unquoted subjects
with a counter, random tokens equal to the message-id, other random
tokens, ...), with examples. LOG.md 2026-10-11 (5).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tools/obf_kinds.ts

@@ -0,0 +1,43 @@
+/**
+ * What the obfuscated lines of a capture are (LOG.md 2026-10-11 (5)):
+ * unquoted subjects with a counter ("Name.exe (1/2)", parseable without
+ * quotes), random tokens equal to the message-id's local part (ngPost-style
+ * private uploads), other random tokens, and the rest; with examples.
+ *
+ *   node --import=@poppinss/ts-exec tools/obf_kinds.ts <capture dir>...
+ */
+import { readFile, readdir } from 'node:fs/promises'
+import { join } from 'node:path'
+import { unzipSync } from 'node:zlib'
+import { SegmentScanner, scanOverviewText } from '#lib/edge/scan'
+
+for (const dir of process.argv.slice(2)) {
+  const { group } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
+  const scanner = new SegmentScanner()
+  const kinds = new Map<string, { n: number; examples: string[] }>()
+  let obfuscated = 0
+  const add = (kind: string, subject: string) => {
+    const k = kinds.get(kind) ?? { n: 0, examples: [] }
+    k.n++
+    if (k.examples.length < 3) k.examples.push(subject.slice(0, 90))
+    kinds.set(kind, k)
+  }
+  for (const file of (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()) {
+    scanOverviewText(unzipSync(await readFile(join(dir, file))).toString('utf8'), (n, subject, from, date, messageId, bytes, xref) => {
+      if (xref && scanner.owner(xref, group, { [group]: 1 }) !== group) return
+      if (scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) return
+      obfuscated++
+      const local = messageId.replace(/^</, '').split('@')[0]
+      const s = subject.trim()
+      if (/\(\d+\/\d+\)\s*$/.test(s) || /\[\d+\/\d+\]\s*$/.test(s)) add(/"/.test(s) ? 'counter, quoted name not parsed' : 'counter, unquoted name', s)
+      else if (s === local) add('random = message-id', s)
+      else if (/^[A-Za-z0-9]{16,}$/.test(s)) add('random token', s)
+      else if (/yEnc/i.test(s)) add('yEnc, no counter', s)
+      else add('other', s)
+    })
+  }
+  console.log(`== ${dir}: ${obfuscated} obfuscated`)
+  for (const [kind, k] of [...kinds].sort((a, b) => b[1].n - a[1].n)) {
+    console.log(`  ${kind}: ${k.n} (${((100 * k.n) / obfuscated).toFixed(1)}%)  e.g. ${k.examples.map((e) => JSON.stringify(e)).join(' | ')}`)
+  }
+}

tools/obf_runs.ts

@@ -0,0 +1,90 @@
+/**
+ * How obfuscated subjects (no quoted file name or no (i/n) counter: the edge
+ * worker only counts them) come in runs, on a captured range: whether a run
+ * of articles can be tracked by article numbers instead of message-ids, and
+ * how many runs there would be (LOG.md 2026-10-11 (5)).
+ *
+ *   node --import=@poppinss/ts-exec tools/obf_runs.ts <capture dir> [gapMinutes]
+ *
+ * Strict runs: consecutive obfuscated articles (in article-number order) of
+ * one poster and posting tool. Keyed runs: per (poster, tool), open while
+ * articles come within gapMinutes of each other, other posts in between.
+ * Writes <capture>/obf-runs.tsv (keyed runs, for the body sample).
+ */
+import { readFile, readdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { unzipSync } from 'node:zlib'
+import { SegmentScanner, scanOverviewText } from '#lib/edge/scan'
+import { uploadTool } from '#lib/rules/index'
+
+const [dir, gapArg] = process.argv.slice(2)
+const gapMs = Number(gapArg ?? 10) * 60_000
+const { group } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8'))
+const scanner = new SegmentScanner()
+const priorities = { [group]: 1 }
+
+interface Run { key: string; poster: string; tool: string; first: number; last: number; articles: number; bytes: number; sizes: Map<number, number>; firstAt: number; lastAt: number; ids: number[] }
+let lines = 0, owned = 0, obfuscated = 0
+let strict = null as Run | null
+const strictRuns: number[] = []
+const open = new Map<string, Run>()
+const keyed: Run[] = []
+const posters = new Set<string>()
+
+const close = (r: Run) => keyed.push(r)
+for (const file of (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()) {
+  const text = unzipSync(await readFile(join(dir, file))).toString('utf8')
+  scanOverviewText(text, (n, subject, from, date, messageId, bytes, xref) => {
+    lines++
+    if (xref && scanner.owner(xref, group, priorities) !== group) return
+    owned++
+    if (scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) {
+      if (strict) strictRuns.push(strict.articles), (strict = null)
+      return
+    }
+    obfuscated++
+    const tool = uploadTool(messageId)
+    const key = `${from}\t${tool}`
+    posters.add(from)
+    const at = Date.parse(date.replace(/ UTC$/, ' GMT')) || 0
+    const size = Number(bytes) || 0
+    if (!strict || strict.key !== key) {
+      if (strict) strictRuns.push(strict.articles)
+      strict = { key, poster: from, tool, first: n, last: n, articles: 0, bytes: 0, sizes: new Map(), firstAt: at, lastAt: at, ids: [] }
+    }
+    strict.articles++
+    let run = open.get(key)
+    if (run && at - run.lastAt > gapMs) {
+      close(run)
+      run = undefined
+    }
+    if (!run) {
+      run = { key, poster: from, tool, first: n, last: n, articles: 0, bytes: 0, sizes: new Map(), firstAt: at, lastAt: at, ids: [] }
+      open.set(key, run)
+    }
+    run.last = n
+    run.articles++
+    run.bytes += size
+    run.sizes.set(size, (run.sizes.get(size) ?? 0) + 1)
+    run.lastAt = Math.max(run.lastAt, at)
+    if (run.ids.length < 3) run.ids.push(n)
+  })
+}
+if (strict !== null) strictRuns.push((strict as Run).articles)
+for (const r of open.values()) close(r)
+
+const pct = (a: number, b: number) => `${((100 * a) / Math.max(1, b)).toFixed(1)}%`
+const quant = (xs: number[], q: number) => xs.slice().sort((a, b) => a - b)[Math.floor(q * (xs.length - 1))] ?? 0
+const keyedSizes = keyed.map((r) => r.articles)
+console.log(`${dir}: ${lines} lines, ${owned} owned, ${obfuscated} obfuscated (${pct(obfuscated, owned)} of owned), ${posters.size} distinct posters among them`)
+console.log(`strict runs: ${strictRuns.length} (${(obfuscated / Math.max(1, strictRuns.length)).toFixed(1)} articles a run; median ${quant(strictRuns, 0.5)}, p90 ${quant(strictRuns, 0.9)})`)
+console.log(`keyed runs (gap ${gapMs / 60000} min): ${keyed.length} (${(obfuscated / Math.max(1, keyed.length)).toFixed(1)} articles a run; median ${quant(keyedSizes, 0.5)}, p90 ${quant(keyedSizes, 0.9)}, max ${quant(keyedSizes, 1)})`)
+const span = keyed.map((r) => (r.last - r.first + 1) / r.articles)
+console.log(`keyed run density (article-number span per article): median ${quant(span, 0.5).toFixed(1)}, p90 ${quant(span, 0.9).toFixed(1)}`)
+// A file posted in equal parts: one size for most articles of a run.
+const modal = keyed.filter((r) => r.articles >= 10).map((r) => Math.max(...r.sizes.values()) / r.articles)
+console.log(`runs of 10+ articles: ${modal.length}; share of their articles at the run's most common size: median ${pct(quant(modal, 0.5), 1)}, p10 ${pct(quant(modal, 0.1), 1)}`)
+const byTool = new Map<string, number>()
+for (const r of keyed) byTool.set(r.tool, (byTool.get(r.tool) ?? 0) + r.articles)
+console.log('obfuscated articles by tool:', [...byTool].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([t, n]) => `${t} ${pct(n, obfuscated)}`).join(', '))
+await writeFile(join(dir, 'obf-runs.tsv'), keyed.sort((a, b) => b.articles - a.articles).map((r) => [r.first, r.last, r.articles, r.bytes, r.tool, r.ids.join(','), r.poster].join('\t')).join('\n') + '\n')