The Rust worker is the product: edge/, built and checked in CI
Test and deploy / test (push) Successful in 46s
Test and deploy / edge (push) Successful in 1m54s
Test and deploy / deploy (push) Successful in 2m56s

User: Rust is the product (option a). tools/edge-bench-rs moves to edge/
(crate edge; binaries edge-node, the worker, and edge-bench, the replay).
CI's new job edge runs cargo test and edge/check-fixtures.sh: both
RangeIndexers on three committed captures (edge/fixtures, 25k overview
lines each of a.b.models, a.b.ath and a.b.boneless in 5 chunks, 3.5 MB),
compared field by field with check.ts; deploy waits for it. The
TypeScript RangeIndexer stays as check.ts's reference. README: what it
is now; the benchmark findings as history.

Random names: looksRandom missed long tokens of capitals and digits
("51294P1CUKR2R35X", release 74539068, readable and a Newznab title).
12+ capitals and digits switching 6+ times between them are random
(COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice); both
implementations, identical on the fixtures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-11 20:01:18 +02:00
1 parent 48f4e982f9
commit 5109481d67
38 files changed
+138 -54

No files matched your search

+144
View File
@@ -0,0 +1,144 @@
//! A batch as the coordinator takes it (lib/edge/wire.ts toWire): JSON with
//! the TypeScript names, chunk payloads in base64.
use crate::indexer::{Batch, Candidate, Fragment, ReleaseChunk};
use base64::Engine;
use serde::Serialize;
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WireRelease<'a> {
key: &'a str,
upload: usize,
name: &'a str,
source: &'a str,
file_total: Option<i64>,
poster: &'a str,
groups: &'a [String],
posted_at: Option<i64>,
posted_last: Option<i64>,
files: usize,
segments: usize,
bytes: i64,
first_article: i64,
last_article: i64,
readable: bool,
has_par2: bool,
payload: String,
#[serde(skip_serializing_if = "Option::is_none")]
rule: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
rule_key: Option<&'a str>,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WireCandidate<'a> {
release_key: &'a str,
upload: usize,
kind: &'a str,
message_id: &'a str,
file: &'a str,
}
#[derive(Serialize)]
struct WireCount<'a> {
rule: &'a str,
key: &'a str,
articles: u64,
files: u64,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WireFragmentRelease<'a> {
key: &'a str,
name: &'a str,
source: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
file_total: Option<i64>,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct WireFragment<'a> {
file_key: &'a str,
name: &'a str,
poster: &'a str,
subject: &'a str,
total_parts: u32,
groups: &'a [String],
release: Option<WireFragmentRelease<'a>>,
posted_at: Option<i64>,
segments: Vec<(u32, &'a str, i64, i64)>,
}
#[derive(Serialize)]
struct WireBatch<'a> {
from: i64,
next: i64,
headers: u64,
releases: Vec<WireRelease<'a>>,
samples: Vec<WireRelease<'a>>,
candidates: Vec<WireCandidate<'a>>,
counts: Vec<WireCount<'a>>,
fragments: Vec<WireFragment<'a>>,
}
fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRelease<'a> {
WireRelease {
key: &r.key,
upload: r.upload,
name: &r.name,
source: r.source,
file_total: r.file_total,
poster: &r.poster,
groups: &r.groups,
posted_at: r.posted_at,
posted_last: r.posted_last,
files: r.files,
segments: r.segments,
bytes: r.bytes,
first_article: r.first_article,
last_article: r.last_article,
readable: r.readable,
has_par2: r.has_par2,
payload: base64::engine::general_purpose::STANDARD.encode(&r.payload),
rule: rule.map(|(rule, _)| rule),
rule_key: rule.map(|(_, key)| key),
}
}
fn candidate(c: &Candidate) -> WireCandidate<'_> {
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
}
fn fragment(f: &Fragment) -> WireFragment<'_> {
WireFragment {
file_key: &f.file_key,
name: &f.name,
poster: &f.poster,
subject: &f.subject,
total_parts: f.total_parts,
groups: &f.groups,
// A named file always has a release; fileTotal only when truthy, as fragmentOf.
release: Some(WireFragmentRelease { key: &f.release.key, name: &f.release.name, source: f.release.source, file_total: f.release.file_total_truthy() }),
posted_at: f.posted_at,
segments: f.segments.iter().map(|(n, id, bytes, article)| (*n, id.as_str(), *bytes, *article)).collect(),
}
}
/// toWire(batch) as JSON bytes.
pub fn to_json(batch: &Batch) -> Vec<u8> {
let wire = WireBatch {
from: batch.from,
next: batch.next,
headers: batch.headers,
releases: batch.releases.iter().map(|r| release(r, None)).collect(),
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
candidates: batch.candidates.iter().map(candidate).collect(),
counts: batch.counts.iter().map(|(rule, key, articles, files)| WireCount { rule, key, articles: *articles, files: *files }).collect(),
fragments: batch.fragments.iter().map(fragment).collect(),
};
serde_json::to_vec(&wire).unwrap()
}