edge-bench-rs: a library for the worker; rules from the lease; the wire format
- src/lib.rs: the indexing modules as a library, for edge-bench-rs and the coming edge-node worker. - Rules as a lease brings them (priorities, dropped signatures and tools, per-article parts, sample share, executables) instead of DEFAULT_RULES hard-coded; RangeIndexer takes them and a resume point. - Fragments carry everything the TypeScript Fragment has (poster, subject, groups, release): the server stores them for the merger. - wire.rs: a batch as toWire makes it (camelCase JSON, payloads in base64). edge-bench-rs --wire writes it; check.ts --wire compares it with toWire of the TypeScript batch: identical on models-a and ath-2013. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fc86dd00d1
commit
dce9967806
8 files changed
+641
-27
No files matched your search
@@ -4,16 +4,12 @@
|
||||
//! idle ones, and every N chunks judge and encode the releases. `--dump`
|
||||
//! writes every output as JSON lines for tools/edge-bench-rs/check.ts.
|
||||
//!
|
||||
//! edge-bench-rs <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl]
|
||||
|
||||
mod codec;
|
||||
mod indexer;
|
||||
mod js;
|
||||
mod names;
|
||||
//! edge-bench-rs <dir> [--batch-chunks 10] [--idle 5] [--dump out.jsonl] [--wire batches.jsonl]
|
||||
|
||||
use base64::Engine;
|
||||
use indexer::{Batch, RangeIndexer};
|
||||
use js::json_str;
|
||||
use edge_bench_rs::indexer::{self, Batch, RangeIndexer};
|
||||
use edge_bench_rs::js::json_str;
|
||||
use edge_bench_rs::{names, wire};
|
||||
use rustc_hash::FxHashMap;
|
||||
use std::io::Write;
|
||||
use std::time::Instant;
|
||||
@@ -159,6 +155,8 @@ fn main() {
|
||||
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
|
||||
let idle: u64 = flag("--idle").map(|v| v.parse().unwrap()).unwrap_or(5);
|
||||
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
// Each batch as the worker posts it (wire.rs), one JSON per line: check.ts compares it with toWire.
|
||||
let mut wire_out = flag("--wire").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
|
||||
|
||||
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
|
||||
let group = json_field(&range, "group");
|
||||
@@ -169,7 +167,7 @@ fn main() {
|
||||
|
||||
let mut priorities = FxHashMap::default();
|
||||
priorities.insert(group.clone(), 1);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, priorities, idle);
|
||||
let mut indexer = RangeIndexer::new(&group, start, end, start, names::Rules::defaults(priorities), idle);
|
||||
|
||||
let (mut inflate_s, mut index_s) = (0.0, 0.0);
|
||||
let (mut compressed, mut text_bytes) = (0usize, 0usize);
|
||||
@@ -212,6 +210,10 @@ fn main() {
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
if let Some(w) = wire_out.as_mut() {
|
||||
w.write_all(&wire::to_json(&batch)).unwrap();
|
||||
w.write_all(b"\n").unwrap();
|
||||
}
|
||||
}
|
||||
let total = inflate_s + index_s;
|
||||
#[allow(static_mut_refs)]
|
||||
|
||||
Reference in new issue
Block a user