edge-bench-rs: libdeflate for chunks, memchr for lines and tabs

libdeflate inflates a chunk whole, in half the time of zlib-rs (1.09 ->
0.51 s per boneless range); memchr finds line ends and tabs with SIMD.
The per-article timers are gone. Boneless 2026 ~700k headers/s, ath 2013
~920k; all three captures still identical to the TypeScript.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 09:34:38 +02:00
1 parent b1c2db5984
commit c7b4ebe470
4 files changed
+47 -25

No files matched your search

+18 -10
View File
@@ -15,21 +15,28 @@ use base64::Engine;
use indexer::{Batch, RangeIndexer};
use js::json_str;
use rustc_hash::FxHashMap;
use std::io::{Read, Write};
use std::io::Write;
use std::time::Instant;
// Many small strings (message-ids, names): the system allocator was a measurable share.
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn inflate(raw: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(raw.len() * 6);
if raw.starts_with(&[0x1f, 0x8b]) {
flate2::read::MultiGzDecoder::new(raw).read_to_end(&mut out).unwrap();
} else {
flate2::read::ZlibDecoder::new(raw).read_to_end(&mut out).unwrap();
/// A chunk as the news server sent it (zlib; gzip also taken), whole: libdeflate wants the output
/// size, so it grows until the text fits (~3.6 times the compressed size on boneless).
fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Vec<u8> {
let mut out = vec![0u8; raw.len() * 4 + 4096];
loop {
let result = if raw.starts_with(&[0x1f, 0x8b]) { d.gzip_decompress(raw, &mut out) } else { d.zlib_decompress(raw, &mut out) };
match result {
Ok(n) => {
out.truncate(n);
return out;
}
Err(libdeflater::DecompressionError::InsufficientSpace) => out.resize(out.len() * 2, 0),
Err(e) => panic!("inflate: {e}"),
}
}
out
}
fn json_field(json: &str, name: &str) -> String {
@@ -166,12 +173,13 @@ fn main() {
let (mut compressed, mut text_bytes) = (0usize, 0usize);
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
let mut keys = std::collections::HashSet::new();
let mut decompressor = libdeflater::Decompressor::new();
let wall = Instant::now();
for (i, file) in files.iter().enumerate() {
let raw = std::fs::read(format!("{dir}/{file}")).unwrap();
compressed += raw.len();
let t = Instant::now();
let data = inflate(&raw);
let data = inflate(&raw, &mut decompressor);
inflate_s += t.elapsed().as_secs_f64();
text_bytes += data.len();
let t = Instant::now();
@@ -206,7 +214,7 @@ fn main() {
let total = inflate_s + index_s;
#[allow(static_mut_refs)]
let p = unsafe { indexer::PROF };
eprintln!("describe {:.2} s ({} calls), dates {:.2} s, aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[0] as f64 / 1e9, p[1], p[2] as f64 / 1e9, p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
eprintln!(" of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000);
println!(
"{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",