From 18b9ffdca3b9701c88b5e7147b755d7a5616cdc9 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 11 Oct 2026 20:43:03 +0200 Subject: [PATCH] tools: obf_runs and obf_kinds, measuring the obfuscated subjects obf_runs: how obfuscated articles come in runs (strict, and per poster and tool with a time gap). obf_kinds: what they are (unquoted subjects with a counter, random tokens equal to the message-id, other random tokens, ...), with examples. LOG.md 2026-10-11 (5). Co-Authored-By: Claude Opus 5.5 --- tools/obf_kinds.ts | 43 ++++++++++++++++++++++ tools/obf_runs.ts | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 tools/obf_kinds.ts create mode 100644 tools/obf_runs.ts diff --git a/tools/obf_kinds.ts b/tools/obf_kinds.ts new file mode 100644 index 0000000..782ce19 --- /dev/null +++ b/tools/obf_kinds.ts @@ -0,0 +1,43 @@ +/** + * What the obfuscated lines of a capture are (LOG.md 2026-10-11 (5)): + * unquoted subjects with a counter ("Name.exe (1/2)", parseable without + * quotes), random tokens equal to the message-id's local part (ngPost-style + * private uploads), other random tokens, and the rest; with examples. + * + * node --import=@poppinss/ts-exec tools/obf_kinds.ts ... + */ +import { readFile, readdir } from 'node:fs/promises' +import { join } from 'node:path' +import { unzipSync } from 'node:zlib' +import { SegmentScanner, scanOverviewText } from '#lib/edge/scan' + +for (const dir of process.argv.slice(2)) { + const { group } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8')) + const scanner = new SegmentScanner() + const kinds = new Map() + let obfuscated = 0 + const add = (kind: string, subject: string) => { + const k = kinds.get(kind) ?? { n: 0, examples: [] } + k.n++ + if (k.examples.length < 3) k.examples.push(subject.slice(0, 90)) + kinds.set(kind, k) + } + for (const file of (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()) { + scanOverviewText(unzipSync(await readFile(join(dir, file))).toString('utf8'), (n, subject, from, date, messageId, bytes, xref) => { + if (xref && scanner.owner(xref, group, { [group]: 1 }) !== group) return + if (scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) return + obfuscated++ + const local = messageId.replace(/^ b[1].n - a[1].n)) { + console.log(` ${kind}: ${k.n} (${((100 * k.n) / obfuscated).toFixed(1)}%) e.g. ${k.examples.map((e) => JSON.stringify(e)).join(' | ')}`) + } +} diff --git a/tools/obf_runs.ts b/tools/obf_runs.ts new file mode 100644 index 0000000..a52cf2a --- /dev/null +++ b/tools/obf_runs.ts @@ -0,0 +1,90 @@ +/** + * How obfuscated subjects (no quoted file name or no (i/n) counter: the edge + * worker only counts them) come in runs, on a captured range: whether a run + * of articles can be tracked by article numbers instead of message-ids, and + * how many runs there would be (LOG.md 2026-10-11 (5)). + * + * node --import=@poppinss/ts-exec tools/obf_runs.ts [gapMinutes] + * + * Strict runs: consecutive obfuscated articles (in article-number order) of + * one poster and posting tool. Keyed runs: per (poster, tool), open while + * articles come within gapMinutes of each other, other posts in between. + * Writes /obf-runs.tsv (keyed runs, for the body sample). + */ +import { readFile, readdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { unzipSync } from 'node:zlib' +import { SegmentScanner, scanOverviewText } from '#lib/edge/scan' +import { uploadTool } from '#lib/rules/index' + +const [dir, gapArg] = process.argv.slice(2) +const gapMs = Number(gapArg ?? 10) * 60_000 +const { group } = JSON.parse(await readFile(join(dir, 'range.json'), 'utf8')) +const scanner = new SegmentScanner() +const priorities = { [group]: 1 } + +interface Run { key: string; poster: string; tool: string; first: number; last: number; articles: number; bytes: number; sizes: Map; firstAt: number; lastAt: number; ids: number[] } +let lines = 0, owned = 0, obfuscated = 0 +let strict = null as Run | null +const strictRuns: number[] = [] +const open = new Map() +const keyed: Run[] = [] +const posters = new Set() + +const close = (r: Run) => keyed.push(r) +for (const file of (await readdir(dir)).filter((f) => f.endsWith('.gz')).sort()) { + const text = unzipSync(await readFile(join(dir, file))).toString('utf8') + scanOverviewText(text, (n, subject, from, date, messageId, bytes, xref) => { + lines++ + if (xref && scanner.owner(xref, group, priorities) !== group) return + owned++ + if (scanner.segment(n, subject, from, date, messageId, bytes, xref, group)) { + if (strict) strictRuns.push(strict.articles), (strict = null) + return + } + obfuscated++ + const tool = uploadTool(messageId) + const key = `${from}\t${tool}` + posters.add(from) + const at = Date.parse(date.replace(/ UTC$/, ' GMT')) || 0 + const size = Number(bytes) || 0 + if (!strict || strict.key !== key) { + if (strict) strictRuns.push(strict.articles) + strict = { key, poster: from, tool, first: n, last: n, articles: 0, bytes: 0, sizes: new Map(), firstAt: at, lastAt: at, ids: [] } + } + strict.articles++ + let run = open.get(key) + if (run && at - run.lastAt > gapMs) { + close(run) + run = undefined + } + if (!run) { + run = { key, poster: from, tool, first: n, last: n, articles: 0, bytes: 0, sizes: new Map(), firstAt: at, lastAt: at, ids: [] } + open.set(key, run) + } + run.last = n + run.articles++ + run.bytes += size + run.sizes.set(size, (run.sizes.get(size) ?? 0) + 1) + run.lastAt = Math.max(run.lastAt, at) + if (run.ids.length < 3) run.ids.push(n) + }) +} +if (strict !== null) strictRuns.push((strict as Run).articles) +for (const r of open.values()) close(r) + +const pct = (a: number, b: number) => `${((100 * a) / Math.max(1, b)).toFixed(1)}%` +const quant = (xs: number[], q: number) => xs.slice().sort((a, b) => a - b)[Math.floor(q * (xs.length - 1))] ?? 0 +const keyedSizes = keyed.map((r) => r.articles) +console.log(`${dir}: ${lines} lines, ${owned} owned, ${obfuscated} obfuscated (${pct(obfuscated, owned)} of owned), ${posters.size} distinct posters among them`) +console.log(`strict runs: ${strictRuns.length} (${(obfuscated / Math.max(1, strictRuns.length)).toFixed(1)} articles a run; median ${quant(strictRuns, 0.5)}, p90 ${quant(strictRuns, 0.9)})`) +console.log(`keyed runs (gap ${gapMs / 60000} min): ${keyed.length} (${(obfuscated / Math.max(1, keyed.length)).toFixed(1)} articles a run; median ${quant(keyedSizes, 0.5)}, p90 ${quant(keyedSizes, 0.9)}, max ${quant(keyedSizes, 1)})`) +const span = keyed.map((r) => (r.last - r.first + 1) / r.articles) +console.log(`keyed run density (article-number span per article): median ${quant(span, 0.5).toFixed(1)}, p90 ${quant(span, 0.9).toFixed(1)}`) +// A file posted in equal parts: one size for most articles of a run. +const modal = keyed.filter((r) => r.articles >= 10).map((r) => Math.max(...r.sizes.values()) / r.articles) +console.log(`runs of 10+ articles: ${modal.length}; share of their articles at the run's most common size: median ${pct(quant(modal, 0.5), 1)}, p10 ${pct(quant(modal, 0.1), 1)}`) +const byTool = new Map() +for (const r of keyed) byTool.set(r.tool, (byTool.get(r.tool) ?? 0) + r.articles) +console.log('obfuscated articles by tool:', [...byTool].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([t, n]) => `${t} ${pct(n, obfuscated)}`).join(', ')) +await writeFile(join(dir, 'obf-runs.tsv'), keyed.sort((a, b) => b.articles - a.articles).map((r) => [r.first, r.last, r.articles, r.bytes, r.tool, r.ids.join(','), r.poster].join('\t')).join('\n') + '\n')