/** * Compressed vs plain XOVER, side by side: each chunk is fetched with * XFEATURE COMPRESS GZIP on one connection and without on another, at the * same moment, so both see the same server load. Prints time and wire bytes * per chunk, and totals. * * node --env-file=.env --import=@poppinss/ts-exec tools/compress_ab.ts [pairs] [articles] * * chunks.tsv: "\t" per line. pairs: chunks in flight at once * (2 connections each, default 3). articles: per chunk (default 50000). */ import { readFileSync } from 'node:fs' import { inflateSync, unzipSync } from 'node:zlib' import { XoverClient } from '#lib/nntp/index' const [file, pairsArg, articlesArg] = process.argv.slice(2) const pairs = Number(pairsArg ?? 3) const articles = Number(articlesArg ?? 50_000) const chunks = readFileSync(file, 'utf8').trim().split('\n').map((l) => { const [group, from] = l.split('\t') return { group, from: Number(from) } }) const env = process.env const connect = (compress: boolean) => XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress, timeoutMs: 180_000 }) interface Result { ms: number; bytes: number; lines: number; error?: string } async function fetch(c: XoverClient, group: string, from: number): Promise { await c.group(group) const at = performance.now() try { const raw = await c.xoverRaw(from, from + articles - 1) const ms = performance.now() - at let text: Buffer try { text = unzipSync(raw) } catch { text = inflateSync(raw) } let lines = 0 for (let i = text.indexOf(10); i !== -1; i = text.indexOf(10, i + 1)) lines++ // The plain reply came as text and was deflated here (level 1): its wire size is the text's. return { ms, bytes: c.compressed ? raw.length : text.length, lines } } catch (e: any) { return { ms: performance.now() - at, bytes: 0, lines: 0, error: e.message } } } const sums = { gz: { ms: 0, bytes: 0, lines: 0 }, plain: { ms: 0, bytes: 0, lines: 0 } } let next = 0 const faster = { gz: 0, plain: 0 } async function lane() { const gz = await connect(true) const plain = await connect(false) if (!gz.compressed) throw new Error('server refused COMPRESS GZIP') while (next < chunks.length) { const { group, from } = chunks[next++] const [a, b] = await Promise.all([fetch(gz, group, from), fetch(plain, group, from)]) const pad = (n: number, w: number) => String(n).padStart(w) console.log( `${group.padEnd(38)} ${pad(from, 12)} gz ${(a.ms / 1000).toFixed(1).padStart(6)} s ${(a.bytes / 1e6).toFixed(2).padStart(6)} MB plain ${(b.ms / 1000).toFixed(1).padStart(6)} s ${(b.bytes / 1e6).toFixed(2).padStart(6)} MB lines ${a.lines}/${b.lines}${a.error ? ` gz: ${a.error}` : ''}${b.error ? ` plain: ${b.error}` : ''}` ) if (a.error || b.error || a.lines !== b.lines) continue sums.gz.ms += a.ms; sums.gz.bytes += a.bytes; sums.gz.lines += a.lines sums.plain.ms += b.ms; sums.plain.bytes += b.bytes; sums.plain.lines += b.lines if (a.ms < b.ms) faster.gz++; else faster.plain++ } gz.close() plain.close() } await Promise.all(Array.from({ length: pairs }, lane)) for (const [k, s] of Object.entries(sums)) { console.log(`${k}: ${(s.lines / (s.ms / 1000)).toFixed(0)} articles/s per connection, ${(s.bytes / s.lines).toFixed(0)} bytes per article, ${((s.bytes * 8) / (s.ms / 1000) / 1e6).toFixed(1)} Mbit/s per connection`) } console.log(`faster: gz ${faster.gz}, plain ${faster.plain}`)