edge-bench-rs: a library for the worker; rules from the lease; the wire format
Test and deploy / test (push) Successful in 26s
Test and deploy / deploy (push) Skipped

- src/lib.rs: the indexing modules as a library, for edge-bench-rs and
  the coming edge-node worker.
- Rules as a lease brings them (priorities, dropped signatures and
  tools, per-article parts, sample share, executables) instead of
  DEFAULT_RULES hard-coded; RangeIndexer takes them and a resume point.
- Fragments carry everything the TypeScript Fragment has (poster,
  subject, groups, release): the server stores them for the merger.
- wire.rs: a batch as toWire makes it (camelCase JSON, payloads in
  base64). edge-bench-rs --wire writes it; check.ts --wire compares it
  with toWire of the TypeScript batch: identical on models-a and
  ath-2013.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 20:27:17 +02:00
1 parent fc86dd00d1
commit dce9967806
8 files changed
+641 -27

No files matched your search

+11 -9
View File
@@ -4,16 +4,12 @@
//! idle ones, and every N chunks judge and encode the releases. `--dump`
//! writes every output as JSON lines for tools/edge-bench-rs/check.ts.
//!
//! edge-bench-rs <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl]
mod codec;
mod indexer;
mod js;
mod names;
//! edge-bench-rs <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl]
use base64::Engine;
use indexer::{Batch, RangeIndexer};
use js::json_str;
use edge_bench_rs::indexer::{self, Batch, RangeIndexer};
use edge_bench_rs::js::json_str;
use edge_bench_rs::{names, wire};
use rustc_hash::FxHashMap;
use std::io::Write;
use std::time::Instant;
@@ -159,6 +155,8 @@ fn main() {
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5);
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
// Each batch as the worker posts it (wire.rs), one JSON per line: check.ts compares it with toWire.
let mut wire_out = flag("--wire").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
let group = json_field(&range, "group");
@@ -169,7 +167,7 @@ fn main() {
let mut priorities = FxHashMap::default();
priorities.insert(group.clone(), 1);
let mut indexer = RangeIndexer::new(&group, start, end, priorities, idle);
let mut indexer = RangeIndexer::new(&group, start, end, start, names::Rules::defaults(priorities), idle);
let (mut inflate_s, mut index_s) = (0.0, 0.0);
let (mut compressed, mut text_bytes) = (0usize, 0usize);
@@ -212,6 +210,10 @@ fn main() {
if let Some(d) = dump.as_mut() {
dump_batch(&batch, d);
}
if let Some(w) = wire_out.as_mut() {
w.write_all(&wire::to_json(&batch)).unwrap();
w.write_all(b"\n").unwrap();
}
}
let total = inflate_s + index_s;
#[allow(static_mut_refs)]