Replay harness: a --to bound and the homelab window driver
edge-bench --group and replay:corpus take --to <article> next to --from, so a replay covers one window of a group's history while the corpus holds others (the 2016 ranges next to 2021's and 2025's). deploy/replay-window.sh runs a window on homelab: every group whose window ranges are all stored is replayed once into usenet_replay through the voyager-3 app image, 4 groups at a time, looping while the corpus fill completes the rest. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
72e973d006
commit
ae735361a0
5 files changed
+99
-12
No files matched your search
+7
-3
@@ -4,7 +4,7 @@
|
||||
//! aggregate files, finish releases through the window, and every N chunks judge and encode them. `--dump`
|
||||
//! writes every output as JSON lines; `--wire` the batches as the worker posts them (`-`: stdout).
|
||||
//!
|
||||
//! edge-bench <range dir> | --group <group dir> [--from article] [--limit ranges] [--rules rules.json]
|
||||
//! edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json]
|
||||
//! [--batch-chunks 10] [--idle 5] [--no-finish] [--dump out.jsonl] [--wire batches.jsonl|-] [--releases out.jsonl]
|
||||
|
||||
use base64::Engine;
|
||||
@@ -211,7 +211,7 @@ const MAX_PENDING_FILES: usize = 100_000;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let usage = "usage: edge-bench <range dir> | --group <group dir> [--from article] [--limit ranges] [--rules rules.json] [--batch-chunks N] [--idle N] [--no-finish] [--dump file] [--wire file|-] [--releases file]";
|
||||
let usage = "usage: edge-bench <range dir> | --group <group dir> [--from article] [--to article] [--limit ranges] [--rules rules.json] [--batch-chunks N] [--idle N] [--no-finish] [--dump file] [--wire file|-] [--releases file]";
|
||||
let flag = |name: &str| args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned();
|
||||
let has = |name: &str| args.iter().any(|a| a == name);
|
||||
let batch_chunks: usize = flag("--batch-chunks").map(|v| v.parse().unwrap()).unwrap_or(10);
|
||||
@@ -233,10 +233,14 @@ fn main() {
|
||||
let (group, ranges) = match flag("--group") {
|
||||
Some(dir) => {
|
||||
let (group, mut ranges) = ranges_in(&dir);
|
||||
// --from: ranges ending before it are skipped (a replay continued); --limit: this many at most.
|
||||
// --from: ranges ending before it are skipped (a replay continued); --to: ranges starting after it (one
|
||||
// window of the group's history, the corpus holding others); --limit: this many at most.
|
||||
if let Some(from) = flag("--from").map(|v| v.parse::<i64>().unwrap()) {
|
||||
ranges.retain(|r| r.end >= from);
|
||||
}
|
||||
if let Some(to) = flag("--to").map(|v| v.parse::<i64>().unwrap()) {
|
||||
ranges.retain(|r| r.start <= to);
|
||||
}
|
||||
if let Some(limit) = flag("--limit").map(|v| v.parse::<usize>().unwrap()) {
|
||||
ranges.truncate(limit);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user