Corpus: stored XOVER chunks for the grouping sweeps
Test and deploy / test (push) Successful in 43s
Test and deploy / edge (push) Successful in 1m48s
Test and deploy / deploy (push) Successful in 1m23s

Ranges marked `capture` keep their XOVER chunks in CORPUS_DIR
(corpus/<group>/<start>-<end>/, edge-bench's layout, range.json once
complete; only chunks that inflate). The prefetcher reads a stored chunk
instead of fetching it, so a reset re-indexes them without the news
server; a chunk a worker reports corrupt goes from the corpus too.

edge-bench gets --releases (release chunks without payload, with their
first subjects) and ships in the edge image; deploy/corpus_sweep.py marks
more ranges (capture), replays the corpus with the deployed build (replay)
and reports suspects and the change since the previous build (report).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-12 09:28:22 +02:00
1 parent d43603a43f
commit 9ea2669b2a
13 files changed
+389 -11

No files matched your search

+36
View File
@@ -57,6 +57,37 @@ fn opt_num(n: Option<i64>, out: &mut String) {
}
}
/// The --releases dump: per release chunk its key, name, rule, counter total, poster, time, counts and first subjects.
fn dump_releases(batch: &Batch, group: &str, out: &mut impl Write) {
let mut line = String::new();
for r in &batch.releases {
line.push_str("{\"group\":");
json_str(group, &mut line);
line.push_str(",\"key\":");
json_str(&r.key, &mut line);
line.push_str(",\"name\":");
json_str(&r.name, &mut line);
line.push_str(&format!(",\"source\":\"{}\",\"fileTotal\":", r.source));
opt_num(r.file_total, &mut line);
line.push_str(",\"poster\":");
json_str(&r.poster, &mut line);
line.push_str(",\"postedAt\":");
opt_num(r.posted_at, &mut line);
line.push_str(",\"postedLast\":");
opt_num(r.posted_last, &mut line);
line.push_str(&format!(",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"hasPar2\":{},\"subjects\":[", r.files, r.segments, r.bytes, r.first_article, r.has_par2));
for (i, s) in r.subjects.iter().enumerate() {
if i > 0 {
line.push(',');
}
json_str(s, &mut line);
}
line.push_str("]}\n");
out.write_all(line.as_bytes()).unwrap();
line.clear();
}
}
fn dump_batch(batch: &Batch, out: &mut impl Write) {
let b64 = base64::engine::general_purpose::STANDARD;
let mut line = String::new();
@@ -157,6 +188,8 @@ fn main() {
let mut dump = flag("--dump").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
// Each batch as the worker posts it (wire.rs), one JSON per line: check.ts compares it with toWire.
let mut wire_out = flag("--wire").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
// One line per release chunk without its payload, with its first subjects (deploy/corpus_sweep.py).
let mut releases_out = flag("--releases").map(|p| std::io::BufWriter::new(std::fs::File::create(p).unwrap()));
let range = std::fs::read_to_string(format!("{dir}/range.json")).unwrap();
let group = json_field(&range, "group");
@@ -214,6 +247,9 @@ fn main() {
w.write_all(&wire::to_json(&batch)).unwrap();
w.write_all(b"\n").unwrap();
}
if let Some(w) = releases_out.as_mut() {
dump_releases(&batch, &group, w);
}
}
let total = inflate_s + index_s;
#[allow(static_mut_refs)]