Daan MeijerandClaude Fable 5.1
aa4022295a
The tiny-archive rule (dropTinyArchives, both indexers): a settled single archive of one article and 2-100 kB under a release-like name, spaces allowed, outside the exempt groups and without an e-book, subtitle, crack, cover or sidecar word: the 2015-17 exe-stub flood in miniature, 1.66M catalog rows (LOG.md (131), (133)). PendingLadder.discard scanned every waiting row against the dropped signatures and classes, 9+ minutes a sweep inside the claim, stalling every lane: now driven from the dropped classes and signatures into the indexes. The namer feeds the indexer the span's overview text chunk by chunk (one string of it topped V8's limit) and gives up spans over 3M articles. Migration 7000: a partial index on the taken probe candidates for the prober's start. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
118 lines
3.3 KiB
Rust
118 lines
3.3 KiB
Rust
//! A lease as the coordinator sends it (lib/edge/lease.ts): the range and the rules, JSON with the TypeScript
|
|
//! names. The edge-node worker takes one per range; edge-bench's replay takes the rules alone (`--rules`).
|
|
|
|
use crate::names::Rules;
|
|
use serde::Deserialize;
|
|
use std::collections::HashMap;
|
|
|
|
#[derive(Deserialize)]
|
|
pub struct LeaseJson {
|
|
pub range: RangeJson,
|
|
pub rules: RulesJson,
|
|
}
|
|
|
|
/// A group lease (lib/edge/lease.ts GroupLease): the group a worker thread holds, the rules, the window's snapshot if any.
|
|
#[derive(Deserialize)]
|
|
pub struct GroupLeaseJson {
|
|
pub group: GroupJson,
|
|
pub rules: RulesJson,
|
|
pub snapshot: Option<SnapshotJson>,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
pub struct GroupJson {
|
|
pub id: i64,
|
|
pub name: String,
|
|
pub priority: i64,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
pub struct SnapshotJson {
|
|
pub next: i64,
|
|
}
|
|
|
|
/// The group's next range (lib/edge/lease.ts NextRange).
|
|
#[derive(Deserialize)]
|
|
pub struct NextRangeJson {
|
|
pub range: RangeSpanJson,
|
|
}
|
|
|
|
#[derive(Deserialize, Clone, Copy)]
|
|
pub struct RangeSpanJson {
|
|
pub id: i64,
|
|
pub start: i64,
|
|
pub end: i64,
|
|
pub next: i64,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
pub struct RangeJson {
|
|
pub id: i64,
|
|
pub group: String,
|
|
pub start: i64,
|
|
pub end: i64,
|
|
pub next: i64,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct RulesJson {
|
|
pub version: i64,
|
|
pub priorities: HashMap<String, u32>,
|
|
#[serde(default)]
|
|
pub dropped_signatures: Vec<String>,
|
|
#[serde(default)]
|
|
pub dropped_classes: Vec<String>,
|
|
#[serde(default)]
|
|
pub dropped_tools: Option<Vec<String>>,
|
|
#[serde(default)]
|
|
pub per_article_max_parts: Option<u32>,
|
|
#[serde(default)]
|
|
pub sample_percent: Option<f64>,
|
|
#[serde(default)]
|
|
pub drop_executables: Option<bool>,
|
|
#[serde(default)]
|
|
pub dropped_posters: Option<Vec<String>>,
|
|
#[serde(default)]
|
|
pub drop_stub_archives: Option<bool>,
|
|
#[serde(default)]
|
|
pub drop_nonsense_pieces: Option<bool>,
|
|
#[serde(default)]
|
|
pub drop_tiny_archives: Option<bool>,
|
|
}
|
|
|
|
impl RulesJson {
|
|
/// The rules with today's defaults under them (a field the stored set doesn't have keeps the default).
|
|
pub fn rules(self) -> Rules {
|
|
let mut rules = Rules::defaults(self.priorities.into_iter().collect());
|
|
rules.version = self.version;
|
|
rules.dropped_signatures = self.dropped_signatures.into_iter().collect();
|
|
rules.dropped_classes = self.dropped_classes.into_iter().collect();
|
|
if let Some(tools) = self.dropped_tools {
|
|
rules.dropped_tools = tools.into_iter().collect();
|
|
}
|
|
if let Some(n) = self.per_article_max_parts {
|
|
rules.per_article_max_parts = n;
|
|
}
|
|
if let Some(p) = self.sample_percent {
|
|
rules.sample_percent = p;
|
|
}
|
|
if let Some(e) = self.drop_executables {
|
|
rules.drop_executables = e;
|
|
}
|
|
if let Some(posters) = self.dropped_posters {
|
|
rules.dropped_posters = posters;
|
|
}
|
|
if let Some(d) = self.drop_stub_archives {
|
|
rules.drop_stub_archives = d;
|
|
}
|
|
if let Some(d) = self.drop_nonsense_pieces {
|
|
rules.drop_nonsense_pieces = d;
|
|
}
|
|
if let Some(d) = self.drop_tiny_archives {
|
|
rules.drop_tiny_archives = d;
|
|
}
|
|
rules
|
|
}
|
|
}
|