Daan MeijerandClaude Fable 5.1
f8cb17903b
The indexer finishes files and releases by posting time instead of the range path: the frontier is the median Date of the last 1,024 plausible dates (within 7 days of it; a run of 1,024 agreeing dates moves it), and a release is finished, judged and written once the frontier is 24 h past its last part, flagged edge when posted within a day of the block's start. At the block's end the rest is written provisionally as edges for the block merger. Files short of parts stay in their release (partial: the store writes complete = false). Gone: fragments and the cut-off, uploads_of (online now), check.ts and the fixture comparison; CI replays the fixtures through edge-bench instead. The wire carries finished, edge and partial. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
115 lines
3.2 KiB
Rust
115 lines
3.2 KiB
Rust
//! A batch as the coordinator takes it (lib/edge/wire.ts toWire): JSON with
|
|
//! the TypeScript names, chunk payloads in base64.
|
|
|
|
use crate::indexer::{Batch, Candidate, ReleaseChunk};
|
|
use base64::Engine;
|
|
use serde::Serialize;
|
|
|
|
#[derive(Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct WireRelease<'a> {
|
|
key: &'a str,
|
|
upload: usize,
|
|
name: &'a str,
|
|
source: &'a str,
|
|
file_total: Option<i64>,
|
|
poster: &'a str,
|
|
groups: &'a [String],
|
|
posted_at: Option<i64>,
|
|
posted_last: Option<i64>,
|
|
files: usize,
|
|
segments: usize,
|
|
bytes: i64,
|
|
first_article: i64,
|
|
last_article: i64,
|
|
readable: bool,
|
|
has_par2: bool,
|
|
finished: bool,
|
|
edge: bool,
|
|
partial: bool,
|
|
payload: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
rule: Option<&'a str>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
rule_key: Option<&'a str>,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct WireCandidate<'a> {
|
|
release_key: &'a str,
|
|
upload: usize,
|
|
kind: &'a str,
|
|
message_id: &'a str,
|
|
file: &'a str,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct WireCount<'a> {
|
|
rule: &'a str,
|
|
key: &'a str,
|
|
articles: u64,
|
|
files: u64,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct WireBatch<'a> {
|
|
from: i64,
|
|
next: i64,
|
|
headers: u64,
|
|
posted_low: Option<i64>,
|
|
posted_high: Option<i64>,
|
|
releases: Vec<WireRelease<'a>>,
|
|
samples: Vec<WireRelease<'a>>,
|
|
candidates: Vec<WireCandidate<'a>>,
|
|
counts: Vec<WireCount<'a>>,
|
|
}
|
|
|
|
fn release<'a>(r: &'a ReleaseChunk, rule: Option<(&'a str, &'a str)>) -> WireRelease<'a> {
|
|
WireRelease {
|
|
key: &r.key,
|
|
upload: r.upload,
|
|
name: &r.name,
|
|
source: r.source,
|
|
file_total: r.file_total,
|
|
poster: &r.poster,
|
|
groups: &r.groups,
|
|
posted_at: r.posted_at,
|
|
posted_last: r.posted_last,
|
|
files: r.files,
|
|
segments: r.segments,
|
|
bytes: r.bytes,
|
|
first_article: r.first_article,
|
|
last_article: r.last_article,
|
|
readable: r.readable,
|
|
has_par2: r.has_par2,
|
|
finished: r.finished,
|
|
edge: r.edge,
|
|
partial: r.partial,
|
|
payload: base64::engine::general_purpose::STANDARD.encode(&r.payload),
|
|
rule: rule.map(|(rule, _)| rule),
|
|
rule_key: rule.map(|(_, key)| key),
|
|
}
|
|
}
|
|
|
|
fn candidate(c: &Candidate) -> WireCandidate<'_> {
|
|
WireCandidate { release_key: &c.release_key, upload: c.upload, kind: c.kind, message_id: &c.message_id, file: &c.file }
|
|
}
|
|
|
|
/// toWire(batch) as JSON bytes.
|
|
pub fn to_json(batch: &Batch) -> Vec<u8> {
|
|
let wire = WireBatch {
|
|
from: batch.from,
|
|
next: batch.next,
|
|
headers: batch.headers,
|
|
posted_low: batch.posted_low,
|
|
posted_high: batch.posted_high,
|
|
releases: batch.releases.iter().map(|r| release(r, None)).collect(),
|
|
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
|
|
candidates: batch.candidates.iter().map(candidate).collect(),
|
|
counts: batch.counts.iter().map(|(rule, key, articles, files)| WireCount { rule, key, articles: *articles, files: *files }).collect(),
|
|
};
|
|
serde_json::to_vec(&wire).unwrap()
|
|
}
|