voyager 3: the window in the Rust edge
Test and deploy / test (push) Successful in 47s
Test and deploy / edge (push) Successful in 1m47s
Test and deploy / deploy (push) Skipped

The indexer finishes files and releases by posting time instead of the
range path: the frontier is the median Date of the last 1,024 plausible
dates (within 7 days of it; a run of 1,024 agreeing dates moves it), and
a release is finished, judged and written once the frontier is 24 h past
its last part, flagged edge when posted within a day of the block's start.
At the block's end the rest is written provisionally as edges for the
block merger. Files short of parts stay in their release (partial: the
store writes complete = false). Gone: fragments and the cut-off,
uploads_of (online now), check.ts and the fixture comparison; CI replays
the fixtures through edge-bench instead. The wire carries finished, edge
and partial.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 21:28:35 +01:00
1 parent 2580f20a43
commit f8cb17903b
13 files changed
+457 -413

No files matched your search

+8 -26
View File
@@ -123,7 +123,7 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
line.push_str(",\"postedLast\":");
opt_num(r.posted_last, line);
line.push_str(&format!(
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"",
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"finished\":{},\"edge\":{},\"partial\":{},\"payload\":\"{}\"",
r.files,
r.segments,
r.bytes,
@@ -131,6 +131,9 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
r.last_article,
r.readable,
r.has_par2,
r.finished,
r.edge,
r.partial,
b64.encode(&r.payload)
));
};
@@ -162,25 +165,6 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}"));
emit(&mut line);
}
for f in &batch.fragments {
line.push_str("{\"t\":\"f\",\"fileKey\":");
json_str(&f.file_key, &mut line);
line.push_str(",\"name\":");
json_str(&f.name, &mut line);
line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts));
opt_num(f.posted_at, &mut line);
line.push_str(",\"segments\":[");
for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() {
if i > 0 {
line.push(',');
}
line.push_str(&format!("[{n},"));
json_str(id, &mut line);
line.push_str(&format!(",{bytes},{article}]"));
}
line.push_str("]}");
emit(&mut line);
}
}
fn main() {
@@ -208,7 +192,7 @@ fn main() {
let (mut inflate_s, mut index_s) = (0.0, 0.0);
let (mut compressed, mut text_bytes) = (0usize, 0usize);
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, samples
let mut keys = std::collections::HashSet::new();
let mut decompressor = libdeflater::Decompressor::new();
let wall = Instant::now();
@@ -242,8 +226,7 @@ fn main() {
out.4 += r.payload.len();
}
out.5 += batch.candidates.len();
out.6 += batch.fragments.len();
out.7 += batch.samples.len();
out.6 += batch.samples.len();
if let Some(d) = dump.as_mut() {
dump_batch(&batch, d);
}
@@ -269,7 +252,7 @@ fn main() {
);
println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb());
println!(
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples",
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} samples",
out.1,
keys.len(),
out.2,
@@ -277,7 +260,6 @@ fn main() {
out.4 as f64 / 1e6,
out.4 as f64 / out.3.max(1) as f64,
out.5,
out.6,
out.7
out.6
);
}