Daan MeijerandClaude Opus 5.5
fc8ec9ff48
Each chunk fetched with and without XFEATURE COMPRESS GZIP on two connections at the same moment. 30 chunks on homelab (24 bottom-500, 6 deferred): medians 2.9 s compressed, 3.2 s plain; 59 vs 350 bytes per article. Slow fetches (27-52 s) hit either mode, rarely both. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
72 lines
3.5 KiB
TypeScript
72 lines
3.5 KiB
TypeScript
/**
|
|
* Compressed vs plain XOVER, side by side: each chunk is fetched with
|
|
* XFEATURE COMPRESS GZIP on one connection and without on another, at the
|
|
* same moment, so both see the same server load. Prints time and wire bytes
|
|
* per chunk, and totals.
|
|
*
|
|
* node --env-file=.env --import=@poppinss/ts-exec tools/compress_ab.ts <chunks.tsv> [pairs] [articles]
|
|
*
|
|
* chunks.tsv: "<group>\t<from>" per line. pairs: chunks in flight at once
|
|
* (2 connections each, default 3). articles: per chunk (default 50000).
|
|
*/
|
|
import { readFileSync } from 'node:fs'
|
|
import { inflateSync, unzipSync } from 'node:zlib'
|
|
import { XoverClient } from '#lib/nntp/index'
|
|
|
|
const [file, pairsArg, articlesArg] = process.argv.slice(2)
|
|
const pairs = Number(pairsArg ?? 3)
|
|
const articles = Number(articlesArg ?? 50_000)
|
|
const chunks = readFileSync(file, 'utf8').trim().split('\n').map((l) => {
|
|
const [group, from] = l.split('\t')
|
|
return { group, from: Number(from) }
|
|
})
|
|
const env = process.env
|
|
const connect = (compress: boolean) =>
|
|
XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress, timeoutMs: 180_000 })
|
|
|
|
interface Result { ms: number; bytes: number; lines: number; error?: string }
|
|
async function fetch(c: XoverClient, group: string, from: number): Promise<Result> {
|
|
await c.group(group)
|
|
const at = performance.now()
|
|
try {
|
|
const raw = await c.xoverRaw(from, from + articles - 1)
|
|
const ms = performance.now() - at
|
|
let text: Buffer
|
|
try { text = unzipSync(raw) } catch { text = inflateSync(raw) }
|
|
let lines = 0
|
|
for (let i = text.indexOf(10); i !== -1; i = text.indexOf(10, i + 1)) lines++
|
|
// The plain reply came as text and was deflated here (level 1): its wire size is the text's.
|
|
return { ms, bytes: c.compressed ? raw.length : text.length, lines }
|
|
} catch (e: any) {
|
|
return { ms: performance.now() - at, bytes: 0, lines: 0, error: e.message }
|
|
}
|
|
}
|
|
|
|
const sums = { gz: { ms: 0, bytes: 0, lines: 0 }, plain: { ms: 0, bytes: 0, lines: 0 } }
|
|
let next = 0
|
|
const faster = { gz: 0, plain: 0 }
|
|
async function lane() {
|
|
const gz = await connect(true)
|
|
const plain = await connect(false)
|
|
if (!gz.compressed) throw new Error('server refused COMPRESS GZIP')
|
|
while (next < chunks.length) {
|
|
const { group, from } = chunks[next++]
|
|
const [a, b] = await Promise.all([fetch(gz, group, from), fetch(plain, group, from)])
|
|
const pad = (n: number, w: number) => String(n).padStart(w)
|
|
console.log(
|
|
`${group.padEnd(38)} ${pad(from, 12)} gz ${(a.ms / 1000).toFixed(1).padStart(6)} s ${(a.bytes / 1e6).toFixed(2).padStart(6)} MB plain ${(b.ms / 1000).toFixed(1).padStart(6)} s ${(b.bytes / 1e6).toFixed(2).padStart(6)} MB lines ${a.lines}/${b.lines}${a.error ? ` gz: ${a.error}` : ''}${b.error ? ` plain: ${b.error}` : ''}`
|
|
)
|
|
if (a.error || b.error || a.lines !== b.lines) continue
|
|
sums.gz.ms += a.ms; sums.gz.bytes += a.bytes; sums.gz.lines += a.lines
|
|
sums.plain.ms += b.ms; sums.plain.bytes += b.bytes; sums.plain.lines += b.lines
|
|
if (a.ms < b.ms) faster.gz++; else faster.plain++
|
|
}
|
|
gz.close()
|
|
plain.close()
|
|
}
|
|
await Promise.all(Array.from({ length: pairs }, lane))
|
|
for (const [k, s] of Object.entries(sums)) {
|
|
console.log(`${k}: ${(s.lines / (s.ms / 1000)).toFixed(0)} articles/s per connection, ${(s.bytes / s.lines).toFixed(0)} bytes per article, ${((s.bytes * 8) / (s.ms / 1000) / 1e6).toFixed(1)} Mbit/s per connection`)
|
|
}
|
|
console.log(`faster: gz ${faster.gz}, plain ${faster.plain}`)
|