edge-bench-rs: libdeflate for chunks, memchr for lines and tabs

libdeflate inflates a chunk whole, in half the time of zlib-rs (1.09 ->
0.51 s per boneless range); memchr finds line ends and tabs with SIMD.
The per-article timers are gone. Boneless 2026 ~700k headers/s, ath 2013
~920k; all three captures still identical to the TypeScript.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tools/edge-bench-rs/Cargo.lock

@@ -120,7 +120,9 @@ dependencies = [
  "base64",
  "chrono",
  "flate2",
+ "libdeflater",
  "md-5",
+ "memchr",
  "mimalloc",
  "regex",
  "rustc-hash",
@@ -239,6 +241,24 @@ version = "0.2.190"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
 
+[[package]]
+name = "libdeflate-sys"
+version = "1.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "libdeflater"
+version = "1.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f"
+dependencies = [
+ "libdeflate-sys",
+]
+
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.49"

tools/edge-bench-rs/Cargo.toml

@@ -8,7 +8,9 @@ description = "The edge worker's indexing (lib/edge RangeIndexer) in Rust, to co
 base64 = "0.22"
 chrono = "0.4"
 flate2 = { version = "1", default-features = false, features = ["zlib-rs"] }
+libdeflater = "1.26.1"
 md-5 = "0.10"
+memchr = "2.8.3"
 mimalloc = "0.1.52"
 regex = "1"
 rustc-hash = "2"

tools/edge-bench-rs/src/indexer.rs

@@ -195,9 +195,7 @@ impl Scanner {
         let file = match self.files.get(self.key.as_str()) {
             Some(f) => f.clone(),
             None => {
-                let t = std::time::Instant::now();
                 let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new);
-                unsafe { PROF[0] += t.elapsed().as_nanos() as u64; PROF[1] += 1; }
                 self.files.insert(self.key.clone(), f.clone());
                 f
             }
@@ -212,7 +210,7 @@ impl Scanner {
             message_id: id.to_string(),
             bytes: js::parse_int(bytes).unwrap_or(0),
             article,
-            posted_at: { let t = std::time::Instant::now(); let d = js::parse_date(date); unsafe { PROF[2] += t.elapsed().as_nanos() as u64; } d },
+            posted_at: js::parse_date(date),
             file,
         })
     }
@@ -244,20 +242,18 @@ fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str,
     let b = text.as_bytes();
     let mut start = 0;
     while start < b.len() {
-        let end = memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len());
+        let end = memchr::memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len());
         let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end };
         if stop > start {
             let l = &text[start..stop];
             let lb = l.as_bytes();
             let mut tabs = [usize::MAX; 9];
             let mut n = 0;
-            for (i, &c) in lb.iter().enumerate() {
-                if c == b'\t' {
-                    tabs[n] = i;
-                    n += 1;
-                    if n == 9 {
-                        break;
-                    }
+            for i in memchr::memchr_iter(b'\t', lb) {
+                tabs[n] = i;
+                n += 1;
+                if n == 9 {
+                    break;
                 }
             }
             if n >= 5 {
@@ -279,10 +275,6 @@ fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str,
     }
 }
 
-fn memchr(needle: u8, haystack: &[u8]) -> Option<usize> {
-    haystack.iter().position(|&c| c == needle)
-}
-
 struct OpenFile {
     parts: Vec<Seg>,
     seen: FxHashSet<u32>,

tools/edge-bench-rs/src/main.rs

@@ -15,21 +15,28 @@ use base64::Engine;
 use indexer::{Batch, RangeIndexer};
 use js::json_str;
 use rustc_hash::FxHashMap;
-use std::io::{Read, Write};
+use std::io::Write;
 use std::time::Instant;
 
 // Many small strings (message-ids, names): the system allocator was a measurable share.
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-fn inflate(raw: &[u8]) -> Vec<u8> {
-    let mut out = Vec::with_capacity(raw.len() * 6);
-    if raw.starts_with(&[0x1f, 0x8b]) {
-        flate2::read::MultiGzDecoder::new(raw).read_to_end(&mut out).unwrap();
-    } else {
-        flate2::read::ZlibDecoder::new(raw).read_to_end(&mut out).unwrap();
+/// A chunk as the news server sent it (zlib; gzip also taken), whole: libdeflate wants the output
+/// size, so it grows until the text fits (~3.6 times the compressed size on boneless).
+fn inflate(raw: &[u8], d: &mut libdeflater::Decompressor) -> Vec<u8> {
+    let mut out = vec![0u8; raw.len() * 4 + 4096];
+    loop {
+        let result = if raw.starts_with(&[0x1f, 0x8b]) { d.gzip_decompress(raw, &mut out) } else { d.zlib_decompress(raw, &mut out) };
+        match result {
+            Ok(n) => {
+                out.truncate(n);
+                return out;
+            }
+            Err(libdeflater::DecompressionError::InsufficientSpace) => out.resize(out.len() * 2, 0),
+            Err(e) => panic!("inflate: {e}"),
+        }
     }
-    out
 }
 
 fn json_field(json: &str, name: &str) -> String {
@@ -166,12 +173,13 @@ fn main() {
     let (mut compressed, mut text_bytes) = (0usize, 0usize);
     let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
     let mut keys = std::collections::HashSet::new();
+    let mut decompressor = libdeflater::Decompressor::new();
     let wall = Instant::now();
     for (i, file) in files.iter().enumerate() {
         let raw = std::fs::read(format!("{dir}/{file}")).unwrap();
         compressed += raw.len();
         let t = Instant::now();
-        let data = inflate(&raw);
+        let data = inflate(&raw, &mut decompressor);
         inflate_s += t.elapsed().as_secs_f64();
         text_bytes += data.len();
         let t = Instant::now();
@@ -206,7 +214,7 @@ fn main() {
     let total = inflate_s + index_s;
     #[allow(static_mut_refs)]
     let p = unsafe { indexer::PROF };
-    eprintln!("describe {:.2} s ({} calls), dates {:.2} s, aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[0] as f64 / 1e9, p[1], p[2] as f64 / 1e9, p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
+    eprintln!("  of which aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9);
     println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000);
     println!(
         "{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)",