Replay harness: a --to bound and the homelab window driver
Test and deploy / test (push) Successful in 52s
Test and deploy / edge (push) Successful in 2m7s
Test and deploy / deploy (push) Skipped

edge-bench --group and replay:corpus take --to <article> next to --from, so a
replay covers one window of a group's history while the corpus holds others
(the 2016 ranges next to 2021's and 2025's). deploy/replay-window.sh runs a
window on homelab: every group whose window ranges are all stored is replayed
once into usenet_replay through the voyager-3 app image, 4 groups at a time,
looping while the corpus fill completes the rest.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-26 07:56:39 +01:00
1 parent 72e973d006
commit ae735361a0
5 files changed
+99 -12

No files matched your search

+7 -3
View File
@@ -4,7 +4,7 @@
//! aggregate files, finish releases through the window, and every N chunks judge and encode them. `--dump`
//! writes every output as JSON lines; `--wire` the batches as the worker posts them (`-`: stdout).
//!
//! edge-bench <range dir> | --group <group dir> [--from article] [--limit ranges] [--rules rules.json]
//! edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json]
//! [--batch-chunks 10] [--idle 5] [--no-finish] [--dump out.jsonl] [--wire batches.jsonl|-] [--releases out.jsonl]
use base64::Engine;
@@ -211,7 +211,7 @@ const MAX_PENDING_FILES: usize = 100_000;
fn main() {
let args: Vec<String> = std::env::args().collect();
let usage = "usage: edge-bench <range dir> | --group <group dir> [--from article] [--limit ranges] [--rules rules.json] [--batch-chunks N] [--idle N] [--no-finish] [--dump file] [--wire file|-] [--releases file]";
let usage = "usage: edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json] [--batch-chunks N] [--idle N] [--no-finish] [--dump file] [--wire file|-] [--releases file]";
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
let has = |name: &str| args.iter().any(|a| a == name);
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
@@ -233,10 +233,14 @@ fn main() {
let (group, ranges) = match flag("--group") {
Some(dir) => {
let (group, mut ranges) = ranges_in(&dir);
// --from: ranges ending before it are skipped (a replay continued); --limit: this many at most.
// --from: ranges ending before it are skipped (a replay continued); --to: ranges starting after it (one
// window of the group's history, the corpus holding others); --limit: this many at most.
if let Some(from) = flag("--from").map(|v| v.parse::<i64>().unwrap()) {
ranges.retain(|r| r.end >= from);
}
if let Some(to) = flag("--to").map(|v| v.parse::<i64>().unwrap()) {
ranges.retain(|r| r.start <= to);
}
if let Some(limit) = flag("--limit").map(|v| v.parse::<usize>().unwrap()) {
ranges.truncate(limit);
}