voyager 3: the window in the Rust edge
The indexer finishes files and releases by posting time instead of the range path: the frontier is the median Date of the last 1,024 plausible dates (within 7 days of it; a run of 1,024 agreeing dates moves it), and a release is finished, judged and written once the frontier is 24 h past its last part, flagged edge when posted within a day of the block's start. At the block's end the rest is written provisionally as edges for the block merger. Files short of parts stay in their release (partial: the store writes complete = false). Gone: fragments and the cut-off, uploads_of (online now), check.ts and the fixture comparison; CI replays the fixtures through edge-bench instead. The wire carries finished, edge and partial. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
2580f20a43
commit
f8cb17903b
13 files changed
+457
-413
No files matched your search
+8
-26
@@ -123,7 +123,7 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
line.push_str(",\"postedLast\":");
|
||||
opt_num(r.posted_last, line);
|
||||
line.push_str(&format!(
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"payload\":\"{}\"",
|
||||
",\"files\":{},\"segments\":{},\"bytes\":{},\"firstArticle\":{},\"lastArticle\":{},\"readable\":{},\"hasPar2\":{},\"finished\":{},\"edge\":{},\"partial\":{},\"payload\":\"{}\"",
|
||||
r.files,
|
||||
r.segments,
|
||||
r.bytes,
|
||||
@@ -131,6 +131,9 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
r.last_article,
|
||||
r.readable,
|
||||
r.has_par2,
|
||||
r.finished,
|
||||
r.edge,
|
||||
r.partial,
|
||||
b64.encode(&r.payload)
|
||||
));
|
||||
};
|
||||
@@ -162,25 +165,6 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
||||
line.push_str(&format!(",\"articles\":{articles},\"files\":{files}}}"));
|
||||
emit(&mut line);
|
||||
}
|
||||
for f in &batch.fragments {
|
||||
line.push_str("{\"t\":\"f\",\"fileKey\":");
|
||||
json_str(&f.file_key, &mut line);
|
||||
line.push_str(",\"name\":");
|
||||
json_str(&f.name, &mut line);
|
||||
line.push_str(&format!(",\"totalParts\":{},\"postedAt\":", f.total_parts));
|
||||
opt_num(f.posted_at, &mut line);
|
||||
line.push_str(",\"segments\":[");
|
||||
for (i, (n, id, bytes, article)) in f.segments.iter().enumerate() {
|
||||
if i > 0 {
|
||||
line.push(',');
|
||||
}
|
||||
line.push_str(&format!("[{n},"));
|
||||
json_str(id, &mut line);
|
||||
line.push_str(&format!(",{bytes},{article}]"));
|
||||
}
|
||||
line.push_str("]}");
|
||||
emit(&mut line);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
@@ -208,7 +192,7 @@ fn main() {
|
||||
|
||||
let (mut inflate_s, mut index_s) = (0.0, 0.0);
|
||||
let (mut compressed, mut text_bytes) = (0usize, 0usize);
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, fragments, samples
|
||||
let mut out = (0u64, 0usize, 0usize, 0usize, 0usize, 0usize, 0usize); // headers, releases, files, segments, payload, candidates, samples
|
||||
let mut keys = std::collections::HashSet::new();
|
||||
let mut decompressor = libdeflater::Decompressor::new();
|
||||
let wall = Instant::now();
|
||||
@@ -242,8 +226,7 @@ fn main() {
|
||||
out.4 += r.payload.len();
|
||||
}
|
||||
out.5 += batch.candidates.len();
|
||||
out.6 += batch.fragments.len();
|
||||
out.7 += batch.samples.len();
|
||||
out.6 += batch.samples.len();
|
||||
if let Some(d) = dump.as_mut() {
|
||||
dump_batch(&batch, d);
|
||||
}
|
||||
@@ -269,7 +252,7 @@ fn main() {
|
||||
);
|
||||
println!(" inflate {inflate_s:.2} s, index {index_s:.2} s; peak RSS {} MB", peak_rss_mb());
|
||||
println!(
|
||||
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} fragments, {} samples",
|
||||
" kept: {} release chunks ({} releases), {} files, {} segments, payload {:.1} MB ({:.1} B/segment); {} candidates, {} samples",
|
||||
out.1,
|
||||
keys.len(),
|
||||
out.2,
|
||||
@@ -277,7 +260,6 @@ fn main() {
|
||||
out.4 as f64 / 1e6,
|
||||
out.4 as f64 / out.3.max(1) as f64,
|
||||
out.5,
|
||||
out.6,
|
||||
out.7
|
||||
out.6
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user