From c7b4ebe470eb3e558815f625ddea69850d15f053 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 09:34:38 +0200 Subject: [PATCH] edge-bench-rs: libdeflate for chunks, memchr for lines and tabs libdeflate inflates a chunk whole, in half the time of zlib-rs (1.09 -> 0.51 s per boneless range); memchr finds line ends and tabs with SIMD. The per-article timers are gone. Boneless 2026 ~700k headers/s, ath 2013 ~920k; all three captures still identical to the TypeScript. Co-Authored-By: Claude Opus 5.5 --- tools/edge-bench-rs/Cargo.lock | 20 ++++++++++++++++++++ tools/edge-bench-rs/Cargo.toml | 2 ++ tools/edge-bench-rs/src/indexer.rs | 22 +++++++--------------- tools/edge-bench-rs/src/main.rs | 28 ++++++++++++++++++---------- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/tools/edge-bench-rs/Cargo.lock b/tools/edge-bench-rs/Cargo.lock index 6419d0e..d7faff5 100644 --- a/tools/edge-bench-rs/Cargo.lock +++ b/tools/edge-bench-rs/Cargo.lock @@ -120,7 +120,9 @@ dependencies = [ "base64", "chrono", "flate2", + "libdeflater", "md-5", + "memchr", "mimalloc", "regex", "rustc-hash", @@ -239,6 +241,24 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +[[package]] +name = "libdeflate-sys" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c" +dependencies = [ + "cc", +] + +[[package]] +name = "libdeflater" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f" +dependencies = [ + "libdeflate-sys", +] + [[package]] name = "libmimalloc-sys" version = "0.1.49" diff --git a/tools/edge-bench-rs/Cargo.toml b/tools/edge-bench-rs/Cargo.toml index 719ec94..0a3f914 100644 --- a/tools/edge-bench-rs/Cargo.toml +++ b/tools/edge-bench-rs/Cargo.toml @@ -8,7 +8,9 @@ description = "The edge worker's indexing (lib/edge RangeIndexer) in Rust, to co base64 = "0.22" chrono = "0.4" flate2 = { version = "1", default-features = false, features = ["zlib-rs"] } +libdeflater = "1.26.1" md-5 = "0.10" +memchr = "2.8.3" mimalloc = "0.1.52" regex = "1" rustc-hash = "2" diff --git a/tools/edge-bench-rs/src/indexer.rs b/tools/edge-bench-rs/src/indexer.rs index f203f6b..a2f7559 100644 --- a/tools/edge-bench-rs/src/indexer.rs +++ b/tools/edge-bench-rs/src/indexer.rs @@ -195,9 +195,7 @@ impl Scanner { let file = match self.files.get(self.key.as_str()) { Some(f) => f.clone(), None => { - let t = std::time::Instant::now(); let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new); - unsafe { PROF[0] += t.elapsed().as_nanos() as u64; PROF[1] += 1; } self.files.insert(self.key.clone(), f.clone()); f } @@ -212,7 +210,7 @@ impl Scanner { message_id: id.to_string(), bytes: js::parse_int(bytes).unwrap_or(0), article, - posted_at: { let t = std::time::Instant::now(); let d = js::parse_date(date); unsafe { PROF[2] += t.elapsed().as_nanos() as u64; } d }, + posted_at: js::parse_date(date), file, }) } @@ -244,20 +242,18 @@ fn scan_overview_text(text: &str, mut line: impl FnMut(Option, &str, &str, let b = text.as_bytes(); let mut start = 0; while start < b.len() { - let end = memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len()); + let end = memchr::memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len()); let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end }; if stop > start { let l = &text[start..stop]; let lb = l.as_bytes(); let mut tabs = [usize::MAX; 9]; let mut n = 0; - for (i, &c) in lb.iter().enumerate() { - if c == b'\t' { - tabs[n] = i; - n += 1; - if n == 9 { - break; - } + for i in memchr::memchr_iter(b'\t', lb) { + tabs[n] = i; + n += 1; + if n == 9 { + break; } } if n >= 5 { @@ -279,10 +275,6 @@ fn scan_overview_text(text: &str, mut line: impl FnMut(Option, &str, &str, } } -fn memchr(needle: u8, haystack: &[u8]) -> Option { - haystack.iter().position(|&c| c == needle) -} - struct OpenFile { parts: Vec, seen: FxHashSet, diff --git a/tools/edge-bench-rs/src/main.rs b/tools/edge-bench-rs/src/main.rs index 5b5f25e..efa8c0b 100644 --- a/tools/edge-bench-rs/src/main.rs +++ b/tools/edge-bench-rs/src/main.rs @@ -15,21 +15,28 @@ use base64::Engine; use indexer::{Batch, RangeIndexer}; use js::json_str; use rustc_hash::FxHashMap; -use std::io::{Read, Write}; +use std::io::Write; use std::time::Instant; // Many small strings (message-ids, names): the system allocator was a measurable share. #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; -fn inflate(raw: &[u8]) -> Vec { - let mut out = Vec::with_capacity(raw.len() * 6); - if raw.starts_with(&[0x1f, 0x8b]) { - flate2::read::MultiGzDecoder::new(raw).read_to_end(&mut out).unwrap(); - } else { - flate2::read::ZlibDecoder::new(raw).read_to_end(&mut out).unwrap(); +/// A chunk as the news server sent it (zlib; gzip also taken), whole: libdeflate wants the output +/// size, so it grows until the text fits (~3.6 times the compressed size on boneless). +fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Vec { + let mut out = vec![0u8; raw.len() * 4 + 4096]; + loop { + let result = if raw.starts_with(&[0x1f, 0x8b]) { d.gzip_decompress(raw, &mut out) } else { d.zlib_decompress(raw, &mut out) }; + match result { + Ok(n) => { + out.truncate(n); + return out; + } + Err(libdeflater::DecompressionError::InsufficientSpace) => out.resize(out.len() * 2, 0), + Err(e) => panic!("inflate: {e}"), + } } - out } fn json_field(json: &str, name: &str) -> String { @@ -166,12 +173,13 @@ fn main() { let (mut compressed, mut text_bytes) = (0usize, 0usize); let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples let mut keys = std::collections::HashSet::new(); + let mut decompressor = libdeflater::Decompressor::new(); let wall = Instant::now(); for (i, file) in files.iter().enumerate() { let raw = std::fs::read(format!("{dir}/{file}")).unwrap(); compressed += raw.len(); let t = Instant::now(); - let data = inflate(&raw); + let data = inflate(&raw, &mut decompressor); inflate_s += t.elapsed().as_secs_f64(); text_bytes += data.len(); let t = Instant::now(); @@ -206,7 +214,7 @@ fn main() { let total = inflate_s + index_s; #[allow(static_mut_refs)] let p = unsafe { indexer::PROF }; - eprintln!("describe {:.2} s ({} calls), dates {:.2} s, aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[0] as f64 / 1e9, p[1], p[2] as f64 / 1e9, p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9); + eprintln!(" of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9); println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000); println!( "{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",