Daan MeijerandClaude Fable 5.1
e470e231d0
The judge gets each file's articles seen and bytes, and whether the window closed on the release. A settled release of one unsplit archive posted to 2+ groups is dropped as a stub archive when its articles are <= 256 KiB with the release >= 1 MB, or it is in 4+ groups, or by a webmail poster (those two under 100 MB); releases touching music, e-book, comics, tabs or fonts groups are exempt. A one-article archive of 100 kB-1 MB under a release-like, unscrambled name without spaces or an e-book format is dropped as a nonsense piece. Both switched by the rule set (dropStubArchives, dropNonsensePieces), mirrored in lib/rules, sampled like every drop (LOG.md (123)-(127)). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1985 lines
83 KiB
Rust
1985 lines
83 KiB
Rust
//! The edge's indexing: lib/edge/scan.ts (SegmentScanner, scanOverviewText) and lib/files/aggregator.ts as in the
|
||
//! TypeScript, then voyager 3's window (REBUILD.md, "Voyager 3: window indexing", 2): files and releases are
|
||
//! finished when the frontier (the median Date of the last FRONTIER_ARTICLES plausible ones) is WINDOW_MS past
|
||
//! their last part, or at the block's end (provisionally, as edges), and written once.
|
||
|
||
use crate::codec::{encode_chunk, ChunkFile};
|
||
use crate::js::{self, sha1_hex};
|
||
use crate::names::{self, is_random_file, looks_scrambled, JudgedFile, Release, Rules, Verdict};
|
||
use rustc_hash::{FxHashMap, FxHashSet};
|
||
use serde::{Deserialize, Serialize};
|
||
use std::rc::Rc;
|
||
|
||
pub static mut PROF: [u64; 8] = [0; 8];
|
||
|
||
pub struct FileMeta {
|
||
pub file_key: String,
|
||
pub name: String,
|
||
pub poster: String,
|
||
pub subject: String,
|
||
pub total_parts: u32,
|
||
pub groups: Vec<String>,
|
||
pub release: Release,
|
||
}
|
||
|
||
pub struct Seg {
|
||
pub file: Rc<FileMeta>,
|
||
pub number: u32,
|
||
pub message_id: String,
|
||
pub bytes: i64,
|
||
pub article: i64,
|
||
pub posted_at: Option<i64>,
|
||
}
|
||
|
||
pub struct ReleaseChunk {
|
||
pub key: String,
|
||
pub upload: usize,
|
||
pub name: String,
|
||
pub source: &'static str,
|
||
pub file_total: Option<i64>,
|
||
pub poster: String,
|
||
pub groups: Vec<String>,
|
||
pub posted_at: Option<i64>,
|
||
pub posted_last: Option<i64>,
|
||
pub files: usize,
|
||
pub segments: usize,
|
||
pub bytes: i64,
|
||
pub first_article: i64,
|
||
pub last_article: i64,
|
||
pub readable: bool,
|
||
pub has_par2: bool,
|
||
/// The window closed on it (false: still open at the block's end, written provisionally).
|
||
pub finished: bool,
|
||
/// Open at the block's end, or within the window of its start: the block merger's.
|
||
pub edge: bool,
|
||
/// A file misses parts.
|
||
pub partial: bool,
|
||
/// The upload tool from the first article's message-id (names::upload_tool), for an unnamed release: with the
|
||
/// group its signature, what the pending ladder learns verdicts by. None for a named one.
|
||
pub tool: Option<&'static str>,
|
||
/// The files' names and segments (codec); empty for an unnamed release (decision f: nothing in the catalog
|
||
/// without a usable name; the pending store keeps its keys and probe targets until a probe names it).
|
||
pub payload: Vec<u8>,
|
||
/// The first files' subjects (at most SUBJECTS): edge-bench's --releases dump for the grouping sweeps.
|
||
pub subjects: Vec<String>,
|
||
}
|
||
|
||
pub const SUBJECTS: usize = 3;
|
||
|
||
pub struct Candidate {
|
||
pub release_key: String,
|
||
pub upload: usize,
|
||
pub kind: &'static str,
|
||
pub message_id: String,
|
||
pub file: String,
|
||
}
|
||
|
||
#[derive(Default)]
|
||
pub struct Batch {
|
||
pub from: i64,
|
||
pub next: i64,
|
||
pub headers: u64,
|
||
/// postedLow / postedHigh: the batch's posting times without the earliest and latest 1% (postedSpan).
|
||
pub posted_low: Option<i64>,
|
||
pub posted_high: Option<i64>,
|
||
pub releases: Vec<ReleaseChunk>,
|
||
pub samples: Vec<(ReleaseChunk, &'static str, String)>,
|
||
pub candidates: Vec<Candidate>,
|
||
/// (rule, key) → (articles, files)
|
||
pub counts: Vec<(&'static str, String, u64, u64)>,
|
||
}
|
||
|
||
struct Counter {
|
||
index: usize,
|
||
length: usize,
|
||
segment: i64,
|
||
total: i64,
|
||
}
|
||
|
||
/// The last "(k/N)" after `after` (anywhere when None: JS -1), scanning back from the end (findCounter).
|
||
fn find_counter(subject: &[u8], after: Option<usize>) -> Option<Counter> {
|
||
let mut open = subject.len();
|
||
loop {
|
||
// lastIndexOf('(', open - 1): at most open - 1.
|
||
if open == 0 {
|
||
return None;
|
||
}
|
||
open = subject[..open].iter().rposition(|&c| c == b'(')?;
|
||
if after.is_some_and(|a| open <= a) {
|
||
return None;
|
||
}
|
||
let mut i = open + 1;
|
||
let mut segment: i64 = 0;
|
||
let mut digits = 0;
|
||
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
||
segment = segment * 10 + (c - b'0') as i64;
|
||
digits += 1;
|
||
i += 1;
|
||
}
|
||
if digits == 0 || subject.get(i) != Some(&b'/') {
|
||
continue;
|
||
}
|
||
i += 1;
|
||
let mut total: i64 = 0;
|
||
digits = 0;
|
||
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
||
total = total * 10 + (c - b'0') as i64;
|
||
digits += 1;
|
||
i += 1;
|
||
}
|
||
if digits == 0 || subject.get(i) != Some(&b')') {
|
||
continue;
|
||
}
|
||
return Some(Counter { index: open, length: i + 1 - open, segment, total });
|
||
}
|
||
}
|
||
|
||
fn xref_groups(xref: &str, group: &str) -> Vec<String> {
|
||
let mut groups = Vec::new();
|
||
for token in js::split_spaces(js::trim(xref)).into_iter().skip(1) {
|
||
if let Some(colon) = token.rfind(':') {
|
||
if colon > 0 {
|
||
groups.push(token[..colon].to_string());
|
||
}
|
||
}
|
||
}
|
||
if groups.is_empty() {
|
||
groups.push(group.to_string());
|
||
}
|
||
groups
|
||
}
|
||
|
||
#[derive(Default)]
|
||
struct Scanner {
|
||
files: FxHashMap<String, Option<Rc<FileMeta>>>,
|
||
key: String,
|
||
}
|
||
|
||
impl Scanner {
|
||
fn owner<'a>(&self, xref: &'a str, group: &'a str, priorities: &FxHashMap<String, u32>) -> &'a str {
|
||
let b = xref.as_bytes();
|
||
let Some(mut at) = xref.find(' ') else { return group };
|
||
let mut owner: Option<&str> = None;
|
||
let mut best = u32::MAX;
|
||
loop {
|
||
while b.get(at) == Some(&b' ') {
|
||
at += 1;
|
||
}
|
||
if at >= b.len() {
|
||
break;
|
||
}
|
||
let end = xref[at..].find(' ').map(|i| i + at);
|
||
// lastIndexOf(':', end ?? length): a colon at or before `end`.
|
||
let limit = end.unwrap_or(b.len());
|
||
let colon = xref[..limit.min(b.len())].rfind(':');
|
||
if let Some(colon) = colon.filter(|&c| c > at) {
|
||
let name = &xref[at..colon];
|
||
if let Some(&p) = priorities.get(name) {
|
||
if p < best {
|
||
best = p;
|
||
owner = Some(name);
|
||
}
|
||
}
|
||
}
|
||
match end {
|
||
Some(e) => at = e,
|
||
None => break,
|
||
}
|
||
}
|
||
owner.unwrap_or(group)
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn segment(&mut self, article: i64, subject: &str, from: &str, date: &str, message_id: &str, bytes: &str, xref: &str, group: &str) -> Option<Seg> {
|
||
if subject.is_empty() || message_id.is_empty() {
|
||
return None;
|
||
}
|
||
let sb = subject.as_bytes();
|
||
// After the last quote; without one, anywhere (unquoted_name finds the file name).
|
||
let name_end = subject.rfind('"');
|
||
let counter = find_counter(sb, name_end)?;
|
||
self.key.clear();
|
||
self.key.push_str(from);
|
||
self.key.push('\u{1}');
|
||
self.key.push_str(&subject[..counter.index]);
|
||
self.key.push('\u{0}');
|
||
self.key.push_str(&subject[counter.index + counter.length..]);
|
||
let file = match self.files.get(self.key.as_str()) {
|
||
Some(f) => f.clone(),
|
||
None => {
|
||
let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new);
|
||
self.files.insert(self.key.clone(), f.clone());
|
||
f
|
||
}
|
||
};
|
||
let file = file?;
|
||
if counter.segment < 1 || counter.segment > file.total_parts as i64 {
|
||
return None;
|
||
}
|
||
let id = if message_id.starts_with('<') { message_id.get(1..message_id.len().saturating_sub(1)).unwrap_or("") } else { message_id };
|
||
Some(Seg {
|
||
number: counter.segment as u32,
|
||
message_id: id.to_string(),
|
||
bytes: js::parse_int(bytes).unwrap_or(0),
|
||
article,
|
||
posted_at: js::parse_date(date),
|
||
file,
|
||
})
|
||
}
|
||
}
|
||
|
||
fn describe(subject: &str, name_end: Option<usize>, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
|
||
let quoted = match name_end {
|
||
Some(end) if end > 0 => subject[..end].rfind('"').map(|start| subject[start + 1..end].to_string()),
|
||
_ => None,
|
||
};
|
||
let name = match quoted {
|
||
Some(name) => name_beside_quotes(&name, &subject[..counter.index]).unwrap_or(name),
|
||
None => unquoted_name(&subject[..counter.index])?,
|
||
};
|
||
let name = name.as_str();
|
||
if name.is_empty() || counter.total < 1 {
|
||
return None;
|
||
}
|
||
let joined = format!("{}{}", &subject[..counter.index], &subject[counter.index + counter.length..]);
|
||
let base = js::trim(&js::collapse_spaces(&joined)).to_string();
|
||
let file_key = sha1_hex(&format!("{from}\n{base}\n{}", counter.total));
|
||
let release = names::release_identity(from, &base, name);
|
||
Some(FileMeta {
|
||
file_key,
|
||
name: name.to_string(),
|
||
poster: from.to_string(),
|
||
subject: base,
|
||
total_parts: counter.total as u32,
|
||
groups: xref_groups(xref, group),
|
||
release,
|
||
})
|
||
}
|
||
|
||
/// EXTENSION: /[^.\s]\.[A-Za-z0-9]{2,4}$/
|
||
fn has_extension(s: &str) -> bool {
|
||
let Some(dot) = s.rfind('.') else { return false };
|
||
let ext = &s.as_bytes()[dot + 1..];
|
||
if !(2..=4).contains(&ext.len()) || !ext.iter().all(|c| c.is_ascii_alphanumeric()) {
|
||
return false;
|
||
}
|
||
s[..dot].chars().last().is_some_and(|c| c != '.' && !js::is_space(c))
|
||
}
|
||
|
||
/// The first / - \d{1,3} - / in `text`: the index of its " - ".
|
||
fn first_track(text: &str) -> Option<usize> {
|
||
let b = text.as_bytes();
|
||
let mut from = 0;
|
||
while let Some(i) = text[from..].find(" - ").map(|i| i + from) {
|
||
let digits = b[i + 3..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if (1..=3).contains(&digits) && b[i + 3 + digits..].starts_with(b" - ") {
|
||
return Some(i);
|
||
}
|
||
from = i + 1;
|
||
}
|
||
None
|
||
}
|
||
|
||
/// postedSpan: posting times without the earliest and latest 1% (floor(n / 100) at each end).
|
||
pub fn posted_span(mut dates: Vec<i64>) -> (Option<i64>, Option<i64>) {
|
||
if dates.is_empty() {
|
||
return (None, None);
|
||
}
|
||
dates.sort_unstable();
|
||
let k = dates.len() / 100;
|
||
(Some(dates[k]), Some(dates[dates.len() - 1 - k]))
|
||
}
|
||
|
||
/// nameBesideQuotes (lib/headers/subject.ts): a file name elsewhere when the quoted text has no extension.
|
||
pub fn name_beside_quotes(quoted: &str, before_counter: &str) -> Option<String> {
|
||
if has_extension(quoted) {
|
||
return None;
|
||
}
|
||
// A stray quote took the last pair: the last quoted part that is a file name.
|
||
let mut named = None;
|
||
let mut rest = before_counter;
|
||
while let Some(open) = rest.find('"') {
|
||
let Some(close) = rest[open + 1..].find('"') else { break };
|
||
let part = &rest[open + 1..open + 1 + close];
|
||
if has_extension(part) {
|
||
named = Some(part);
|
||
}
|
||
rest = &rest[open + 1 + close + 1..];
|
||
}
|
||
if let Some(part) = named {
|
||
return Some(part.to_string());
|
||
}
|
||
// .replace(/"[^"]*"/g, ' ')
|
||
let mut text = String::with_capacity(before_counter.len());
|
||
let mut rest = before_counter;
|
||
while let Some(open) = rest.find('"') {
|
||
match rest[open + 1..].find('"') {
|
||
Some(close) => {
|
||
text.push_str(&rest[..open]);
|
||
text.push(' ');
|
||
rest = &rest[open + 1 + close + 1..];
|
||
}
|
||
None => break,
|
||
}
|
||
}
|
||
text.push_str(rest);
|
||
unquoted_name(&text).filter(|n| has_extension(n))
|
||
}
|
||
|
||
/// unquotedName (lib/headers/subject.ts): the file name of a subject without a quoted one, from the text before its counter.
|
||
/// /\.(jpe?g|png|gif|bmp|webp|tiff?)$/i
|
||
fn is_image_name(name: &str) -> bool {
|
||
let lower = name.to_ascii_lowercase();
|
||
[".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tif", ".tiff"].iter().any(|e| lower.ends_with(e))
|
||
}
|
||
|
||
/// DATE_ONLY (lib/headers/subject.ts): a file name that is a date alone ("15_07_2015.rar").
|
||
fn date_only(name: &str) -> bool {
|
||
static DATE_ONLY: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
|
||
regex::Regex::new(r"^(?:[0-9]{1,2}[-_.][0-9]{1,2}[-_.](?:19|20)[0-9]{2}|(?:19|20)[0-9]{2}[-_.][0-9]{1,2}[-_.][0-9]{1,2})\.[A-Za-z0-9]{2,4}$").unwrap()
|
||
});
|
||
DATE_ONLY.is_match(name)
|
||
}
|
||
|
||
pub fn unquoted_name(before_counter: &str) -> Option<String> {
|
||
let mut text = before_counter.trim_end_matches(js::is_space);
|
||
if text.len() >= 4 && text.is_char_boundary(text.len() - 4) && text[text.len() - 4..].eq_ignore_ascii_case("yenc") {
|
||
text = text[..text.len() - 4].trim_end_matches(js::is_space);
|
||
}
|
||
if let Some(quote) = text.find('"') {
|
||
if text[quote + 1..].contains('"') {
|
||
return None;
|
||
}
|
||
let before = &text[..quote];
|
||
if let Some(bracket) = before.rfind('[') {
|
||
let inside = js::trim(&before[bracket + 1..]);
|
||
// replace(/^\d+\/\d+ +/, '')
|
||
let b = inside.as_bytes();
|
||
let mut i = 0;
|
||
while i < b.len() && b[i].is_ascii_digit() {
|
||
i += 1;
|
||
}
|
||
let mut stripped = inside;
|
||
if i > 0 && b.get(i) == Some(&b'/') {
|
||
let mut j = i + 1;
|
||
while j < b.len() && b[j].is_ascii_digit() {
|
||
j += 1;
|
||
}
|
||
if j > i + 1 && b.get(j) == Some(&b' ') {
|
||
while j < b.len() && b[j] == b' ' {
|
||
j += 1;
|
||
}
|
||
stripped = &inside[j..];
|
||
}
|
||
}
|
||
if has_extension(stripped) {
|
||
return Some(stripped.to_string());
|
||
}
|
||
}
|
||
let word = &before[before.rfind(' ').map_or(0, |i| i + 1)..];
|
||
return has_extension(word).then(|| word.to_string());
|
||
}
|
||
// A single-quoted name at the end: /'([^']*[^.\s']\.[A-Za-z0-9]{2,4})'$/
|
||
if let Some(inner) = text.strip_suffix('\'') {
|
||
if let Some(open) = inner.rfind('\'') {
|
||
let part = &inner[open + 1..];
|
||
if has_extension(part) {
|
||
return Some(part.to_string());
|
||
}
|
||
}
|
||
}
|
||
if has_extension(text) {
|
||
let bracket = text.rfind("] ").map_or(0, |i| i + 2);
|
||
// A track or part number (/ - \d{1,3} - /): the name starts there; else after the last " - ".
|
||
let dash = first_track(text).or_else(|| text.rfind(" - ")).map_or(0, |i| i + 3);
|
||
// Or after an arrow ("… -> Exclusive-tgirls_000000-lg.jpg").
|
||
let arrow = text.rfind("-> ").map_or(0, |i| i + 3);
|
||
let cut = bracket.max(dash).max(arrow);
|
||
// A date alone after the cut ("… [ettv] 15_07_2015.rar"): the whole text is the name.
|
||
let name = if date_only(js::trim(&text[cut..])) { js::trim(text) } else { js::trim(&text[cut..]) };
|
||
// An image after numbered text without a separator: its last word ("<abm> … AB: part2 0573-1002 043.jpg").
|
||
// Not an index alone ("Teen Tease Self Shooter sad1 (10).jpg").
|
||
if cut == 0 && is_image_name(name) && name.contains(' ') {
|
||
let space = name.rfind(' ').unwrap();
|
||
let last = &name[space + 1..];
|
||
if name[..space].trim_end_matches(js::is_space).ends_with(|c: char| c.is_ascii_digit()) && !last.starts_with('(') {
|
||
return Some(last.to_string());
|
||
}
|
||
}
|
||
// Ending in a host is no file name: the one in brackets or a word follows.
|
||
if !is_host(name) {
|
||
return (!name.is_empty() && has_extension(name)).then(|| name.to_string());
|
||
}
|
||
}
|
||
// A name in brackets: the last /\[([^[\]]*[^.\s[\]]\.[A-Za-z0-9]{2,4})\]/.
|
||
let mut bracketed = None;
|
||
for (open, _) in text.match_indices('[') {
|
||
let inside = &text[open + 1..];
|
||
let Some(close) = inside.find(']') else { break };
|
||
let part = &inside[..close];
|
||
if !part.contains('[') && has_extension(part) {
|
||
bracketed = Some(part);
|
||
}
|
||
}
|
||
if let Some(part) = bracketed {
|
||
return Some(part.to_string());
|
||
}
|
||
text.split(' ').find(|w| has_extension(w) && !is_host(w)).map(str::to_string)
|
||
}
|
||
|
||
/// /www\./i: a host ("www.libranews.nl"), whose ".nl" is no extension.
|
||
fn is_host(s: &str) -> bool {
|
||
s.to_ascii_lowercase().contains("www.")
|
||
}
|
||
|
||
/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText).
|
||
fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str, &str, &str, &str, &str)) {
|
||
let b = text.as_bytes();
|
||
let mut start = 0;
|
||
while start < b.len() {
|
||
let end = memchr::memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len());
|
||
let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end };
|
||
if stop > start {
|
||
let l = &text[start..stop];
|
||
let lb = l.as_bytes();
|
||
let mut tabs = [usize::MAX; 9];
|
||
let mut n = 0;
|
||
for i in memchr::memchr_iter(b'\t', lb) {
|
||
tabs[n] = i;
|
||
n += 1;
|
||
if n == 9 {
|
||
break;
|
||
}
|
||
}
|
||
if n >= 5 {
|
||
let field = |a: usize, b: usize| -> &str {
|
||
if a == usize::MAX {
|
||
""
|
||
} else {
|
||
&l[a + 1..if b == usize::MAX { l.len() } else { b }]
|
||
}
|
||
};
|
||
let mut xref = if n >= 8 { field(tabs[7], tabs[8]) } else { "" };
|
||
if let Some(rest) = xref.strip_prefix("Xref:") {
|
||
xref = rest.strip_prefix(' ').unwrap_or(rest);
|
||
}
|
||
line(js::parse_int(&l[..tabs[0]]), &l[tabs[0] + 1..tabs[1]], field(tabs[1], tabs[2]), field(tabs[2], tabs[3]), field(tabs[3], tabs[4]), field(tabs[5], tabs[6]), xref);
|
||
}
|
||
}
|
||
start = end + 1;
|
||
}
|
||
}
|
||
|
||
/// A re-upload's copy of a file is kept apart (FileAggregator, uploadGapMs); a file joins the open release of its
|
||
/// key posted within this gap of it, else starts one (uploadsOf, online).
|
||
pub const UPLOAD_GAP_MS: i64 = 3 * 60 * 60 * 1000;
|
||
/// The window (REBUILD.md, "Voyager 3: window indexing", 2): a file still missing parts, and a release, are finished
|
||
/// once the frontier is this far past their last part.
|
||
pub const WINDOW_MS: i64 = 24 * 60 * 60 * 1000;
|
||
/// The frontier is the median Date of the last this many plausibly dated articles: enough that a minority of skewed
|
||
/// dates can't move it, few enough that it trails the newest article by little even in a quiet group (half the
|
||
/// ring: a day at 500 articles a day).
|
||
pub const FRONTIER_ARTICLES: usize = 1024;
|
||
/// A Date further than this from the frontier isn't believed (clock skew, bogus years); FRONTIER_ARTICLES of them
|
||
/// in a row that agree among themselves move it (the group paused for longer).
|
||
pub const PLAUSIBLE_MS: i64 = 7 * 24 * 60 * 60 * 1000;
|
||
/// The frontier is established once this many dated articles were seen (a lone bogus date doesn't set it).
|
||
const FRONTIER_MIN: usize = 64;
|
||
/// Albums (REBUILD.md, "Voyager 3: window indexing", 3): a poster's single-file releases whose file names differ only
|
||
/// by number sequences, posted within WINDOW_MS of the album's first, written as one release (`source: "album"`); an
|
||
/// album holds at most this many files (a dump of 38k pictures a day is a few albums, not one).
|
||
pub const ALBUM_MAX_FILES: usize = 1000;
|
||
/// A counter total this large, shared by this many releases of one poster, is a batch counter (Yenc-PP-A&A's
|
||
/// [n/68067] over thousands of different programs), not a file count: those releases get no file total.
|
||
pub const BATCH_COUNTER_MIN_TOTAL: i64 = 100;
|
||
pub const BATCH_COUNTER_RELEASES: u32 = 20;
|
||
/// Batches after which a poster's counter total is forgotten (BATCH_COUNTER_RELEASES must come within them).
|
||
const BATCH_COUNTER_BATCHES: u64 = 100;
|
||
|
||
fn median(dates: &mut [i64]) -> i64 {
|
||
let mid = dates.len() / 2;
|
||
*dates.select_nth_unstable(mid).1
|
||
}
|
||
|
||
/// Where the block is in posting time: the median Date of the last FRONTIER_ARTICLES plausible ones. Date headers
|
||
/// alone aren't trusted: a date more than PLAUSIBLE_MS from the frontier is set aside, and only FRONTIER_ARTICLES
|
||
/// such dates in a row, agreeing among themselves, move the frontier there.
|
||
#[derive(Default, Serialize, Deserialize)]
|
||
pub struct Frontier {
|
||
ring: Vec<i64>,
|
||
at: usize,
|
||
value: Option<i64>,
|
||
/// The frontier when first established: the block's start in posting time (releases within WINDOW_MS of it are edges).
|
||
start: Option<i64>,
|
||
/// Dates set aside since the last plausible one.
|
||
aside: Vec<i64>,
|
||
pending: usize,
|
||
}
|
||
|
||
impl Frontier {
|
||
pub fn add(&mut self, t: i64) {
|
||
if let Some(f) = self.value {
|
||
if (t - f).abs() > PLAUSIBLE_MS {
|
||
self.aside.push(t);
|
||
if self.aside.len() >= FRONTIER_ARTICLES {
|
||
let m = median(&mut self.aside.clone());
|
||
if self.aside.iter().filter(|&&d| (d - m).abs() <= PLAUSIBLE_MS).count() * 2 >= self.aside.len() {
|
||
self.ring = self.aside.iter().copied().filter(|&d| (d - m).abs() <= PLAUSIBLE_MS).collect();
|
||
self.ring.truncate(FRONTIER_ARTICLES);
|
||
self.at = self.ring.len() % FRONTIER_ARTICLES;
|
||
self.value = Some(m);
|
||
}
|
||
self.aside.clear();
|
||
}
|
||
return;
|
||
}
|
||
self.aside.clear();
|
||
}
|
||
if self.ring.len() < FRONTIER_ARTICLES {
|
||
self.ring.push(t);
|
||
} else {
|
||
self.ring[self.at] = t;
|
||
}
|
||
self.at = (self.at + 1) % FRONTIER_ARTICLES;
|
||
self.pending += 1;
|
||
}
|
||
|
||
/// Recomputes the frontier (once per chunk: a median of FRONTIER_ARTICLES isn't for every article).
|
||
pub fn update(&mut self) {
|
||
if self.pending == 0 || (self.value.is_none() && self.ring.len() < FRONTIER_MIN) {
|
||
return;
|
||
}
|
||
self.pending = 0;
|
||
let m = median(&mut self.ring.clone());
|
||
self.value = Some(m);
|
||
if self.start.is_none() {
|
||
self.start = Some(m);
|
||
}
|
||
}
|
||
|
||
/// At the block's end: whatever dates there are set the frontier (a small block never reached FRONTIER_MIN).
|
||
pub fn settle(&mut self) {
|
||
if self.value.is_none() && !self.ring.is_empty() {
|
||
self.pending = 1;
|
||
let m = median(&mut self.ring.clone());
|
||
self.value = Some(m);
|
||
self.start = Some(m);
|
||
} else {
|
||
self.update();
|
||
}
|
||
}
|
||
|
||
pub fn value(&self) -> Option<i64> {
|
||
self.value
|
||
}
|
||
|
||
pub fn start(&self) -> Option<i64> {
|
||
self.start
|
||
}
|
||
|
||
/// The posting time before which a last part means the window closed.
|
||
fn closed_before(&self) -> Option<i64> {
|
||
self.value.map(|f| f - WINDOW_MS)
|
||
}
|
||
}
|
||
|
||
struct OpenFile {
|
||
parts: Vec<Seg>,
|
||
seen: FxHashSet<u32>,
|
||
total: u32,
|
||
touched: u64,
|
||
/// Posting times seen.
|
||
earliest: Option<i64>,
|
||
latest: Option<i64>,
|
||
}
|
||
|
||
// The snapshot (REBUILD.md, "Voyager 3: window indexing", 5): the window's state between two batches, serialized
|
||
// (bincode, gzip by the worker) and kept by the coordinator, restored by the group's next holder. Segments share
|
||
// their file's FileMeta through an Rc; in the snapshot the files are a table and a segment points at its index.
|
||
#[derive(Serialize, Deserialize)]
|
||
struct SegSnap {
|
||
file: u32,
|
||
number: u32,
|
||
message_id: String,
|
||
bytes: i64,
|
||
article: i64,
|
||
posted_at: Option<i64>,
|
||
}
|
||
|
||
#[derive(Serialize, Deserialize)]
|
||
struct OpenFileSnap {
|
||
parts: Vec<SegSnap>,
|
||
total: u32,
|
||
touched: u64,
|
||
earliest: Option<i64>,
|
||
latest: Option<i64>,
|
||
}
|
||
|
||
#[derive(Serialize, Deserialize)]
|
||
struct OpenReleaseSnap {
|
||
upload: usize,
|
||
files: Vec<Vec<SegSnap>>,
|
||
first: Option<i64>,
|
||
last: Option<i64>,
|
||
touched: u64,
|
||
}
|
||
|
||
#[derive(Serialize, Deserialize)]
|
||
struct OpenAlbumSnap {
|
||
upload: usize,
|
||
members: Vec<(String, OpenReleaseSnap)>,
|
||
first: Option<i64>,
|
||
last: Option<i64>,
|
||
touched: u64,
|
||
}
|
||
|
||
/// A file's details in the snapshot (FileMeta, its release identity's source as text: a &'static str in memory).
|
||
#[derive(Serialize, Deserialize)]
|
||
struct FileMetaSnap {
|
||
file_key: String,
|
||
name: String,
|
||
poster: String,
|
||
subject: String,
|
||
total_parts: u32,
|
||
groups: Vec<String>,
|
||
release_key: String,
|
||
release_name: String,
|
||
release_source: String,
|
||
release_file_total: Option<i64>,
|
||
}
|
||
|
||
/// The sources release_identity gives (lib/release/source.ts), as the static strings the code compares.
|
||
fn source_static(source: &str) -> &'static str {
|
||
match source {
|
||
"subject" => "subject",
|
||
"post" => "post",
|
||
"stem" => "stem",
|
||
"album" => "album",
|
||
"weak-stem" => "weak-stem",
|
||
_ => "stem",
|
||
}
|
||
}
|
||
|
||
#[derive(Serialize, Deserialize)]
|
||
pub struct Snapshot {
|
||
pub version: u32,
|
||
pub group: String,
|
||
/// The article the window stands at: the next one to index (a batch boundary).
|
||
pub next: i64,
|
||
frontier: Frontier,
|
||
files: Vec<FileMetaSnap>,
|
||
aggregator: Vec<(String, Vec<OpenFileSnap>)>,
|
||
aggregator_batch: u64,
|
||
open: Vec<(String, Vec<OpenReleaseSnap>)>,
|
||
albums: Vec<(String, Vec<OpenAlbumSnap>)>,
|
||
uploads: Vec<(String, usize)>,
|
||
batch_totals: Vec<(String, i64, u32, u64)>,
|
||
}
|
||
|
||
/// The snapshot's format version: raised when the state's shape changes (an older snapshot is refused: the group
|
||
/// continues at its resume point, one seam).
|
||
pub const SNAPSHOT_VERSION: u32 = 1;
|
||
|
||
/// The files of a snapshot as they are taken: each FileMeta once, by its Rc.
|
||
#[derive(Default)]
|
||
struct FileTable {
|
||
index: FxHashMap<*const FileMeta, u32>,
|
||
files: Vec<Rc<FileMeta>>,
|
||
}
|
||
|
||
impl FileTable {
|
||
fn seg(&mut self, s: &Seg) -> SegSnap {
|
||
let ptr = Rc::as_ptr(&s.file);
|
||
let file = match self.index.get(&ptr) {
|
||
Some(&i) => i,
|
||
None => {
|
||
let i = self.files.len() as u32;
|
||
self.index.insert(ptr, i);
|
||
self.files.push(s.file.clone());
|
||
i
|
||
}
|
||
};
|
||
SegSnap { file, number: s.number, message_id: s.message_id.clone(), bytes: s.bytes, article: s.article, posted_at: s.posted_at }
|
||
}
|
||
|
||
fn segs(&mut self, segs: &[Seg]) -> Vec<SegSnap> {
|
||
segs.iter().map(|s| self.seg(s)).collect()
|
||
}
|
||
|
||
fn release(&mut self, u: &OpenRelease) -> OpenReleaseSnap {
|
||
OpenReleaseSnap { upload: u.upload, files: u.files.iter().map(|f| self.segs(f)).collect(), first: u.first, last: u.last, touched: u.touched }
|
||
}
|
||
|
||
/// The table's files as the snapshot stores them.
|
||
fn into_files(self) -> Vec<FileMetaSnap> {
|
||
self.files
|
||
.iter()
|
||
.map(|f| FileMetaSnap {
|
||
file_key: f.file_key.clone(),
|
||
name: f.name.clone(),
|
||
poster: f.poster.clone(),
|
||
subject: f.subject.clone(),
|
||
total_parts: f.total_parts,
|
||
groups: f.groups.clone(),
|
||
release_key: f.release.key.clone(),
|
||
release_name: f.release.name.clone(),
|
||
release_source: f.release.source.to_string(),
|
||
release_file_total: f.release.file_total,
|
||
})
|
||
.collect()
|
||
}
|
||
}
|
||
|
||
fn file_of(f: FileMetaSnap) -> FileMeta {
|
||
FileMeta {
|
||
file_key: f.file_key,
|
||
name: f.name,
|
||
poster: f.poster,
|
||
subject: f.subject,
|
||
total_parts: f.total_parts,
|
||
groups: f.groups,
|
||
release: Release { key: f.release_key, name: f.release_name, source: source_static(&f.release_source), file_total: f.release_file_total },
|
||
}
|
||
}
|
||
|
||
fn seg_of(s: SegSnap, files: &[Rc<FileMeta>]) -> Seg {
|
||
Seg { file: files[s.file as usize].clone(), number: s.number, message_id: s.message_id, bytes: s.bytes, article: s.article, posted_at: s.posted_at }
|
||
}
|
||
|
||
fn release_of(u: OpenReleaseSnap, files: &[Rc<FileMeta>]) -> OpenRelease {
|
||
OpenRelease { upload: u.upload, files: u.files.into_iter().map(|f| f.into_iter().map(|s| seg_of(s, files)).collect()).collect(), first: u.first, last: u.last, touched: u.touched }
|
||
}
|
||
|
||
impl OpenFile {
|
||
fn distance(&self, t: Option<i64>) -> i64 {
|
||
match (t, self.earliest, self.latest) {
|
||
(Some(t), Some(e), Some(l)) => 0.max(e - t).max(t - l),
|
||
_ => 0,
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Default)]
|
||
struct Aggregator {
|
||
/// Per file key its open copies, oldest first.
|
||
open: FxHashMap<String, Vec<OpenFile>>,
|
||
batch: u64,
|
||
}
|
||
|
||
impl Aggregator {
|
||
fn add(&mut self, segments: Vec<Seg>) -> Vec<Vec<Seg>> {
|
||
self.batch += 1;
|
||
let batch = self.batch;
|
||
let mut touched: Vec<String> = Vec::new();
|
||
let mut touched_set: FxHashSet<String> = FxHashSet::default();
|
||
for s in segments {
|
||
let key = s.file.file_key.clone();
|
||
if !touched_set.contains(key.as_str()) {
|
||
touched_set.insert(key.clone());
|
||
touched.push(key.clone());
|
||
}
|
||
let copies = self.open.entry(key).or_default();
|
||
let t = s.posted_at;
|
||
// A copy lacking the part, the nearest in time.
|
||
let mut chosen: Option<usize> = None;
|
||
let mut best = i64::MAX;
|
||
for (i, copy) in copies.iter().enumerate() {
|
||
if copy.seen.contains(&s.number) {
|
||
continue;
|
||
}
|
||
let d = copy.distance(t);
|
||
if d < best {
|
||
best = d;
|
||
chosen = Some(i);
|
||
}
|
||
}
|
||
let index = match chosen {
|
||
Some(i) => i,
|
||
None => {
|
||
if let Some(near) = copies.iter_mut().find(|c| c.distance(t) <= UPLOAD_GAP_MS) {
|
||
near.touched = batch;
|
||
continue;
|
||
}
|
||
copies.push(OpenFile { parts: Vec::new(), seen: FxHashSet::default(), total: s.file.total_parts, touched: batch, earliest: None, latest: None });
|
||
copies.len() - 1
|
||
}
|
||
};
|
||
let file = &mut copies[index];
|
||
file.seen.insert(s.number);
|
||
if let Some(t) = t {
|
||
if file.earliest.is_none_or(|e| t < e) {
|
||
file.earliest = Some(t);
|
||
}
|
||
if file.latest.is_none_or(|l| t > l) {
|
||
file.latest = Some(t);
|
||
}
|
||
}
|
||
file.touched = batch;
|
||
file.parts.push(s);
|
||
}
|
||
let mut complete = Vec::new();
|
||
for key in touched {
|
||
let copies = self.open.get_mut(&key).unwrap();
|
||
let mut i = 0;
|
||
while i < copies.len() {
|
||
if (copies[i].parts.len() as u32) < copies[i].total {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
let mut f = copies.remove(i);
|
||
f.parts.sort_by_key(|s| s.number);
|
||
complete.push(f.parts);
|
||
}
|
||
if copies.is_empty() {
|
||
self.open.remove(&key);
|
||
}
|
||
}
|
||
complete
|
||
}
|
||
|
||
/// Files the window closed on, still missing parts: those whose last part is before `closed_before`, undated ones
|
||
/// idle for `idle` batches (all when `all`). Their segments, part order.
|
||
fn evict(&mut self, closed_before: Option<i64>, idle: u64, all: bool) -> Vec<Vec<Seg>> {
|
||
let batch = self.batch;
|
||
let mut out = Vec::new();
|
||
let keys: Vec<String> = self.open.keys().cloned().collect();
|
||
for key in keys {
|
||
let copies = self.open.get_mut(&key).unwrap();
|
||
let mut i = 0;
|
||
while i < copies.len() {
|
||
let c = &copies[i];
|
||
let closed = match c.latest {
|
||
Some(l) => closed_before.is_some_and(|before| l < before),
|
||
None => idle > 0 && batch - c.touched >= idle,
|
||
};
|
||
if !all && !closed {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
let mut f = copies.remove(i);
|
||
f.parts.sort_by_key(|s| s.number);
|
||
out.push(f.parts);
|
||
}
|
||
if copies.is_empty() {
|
||
self.open.remove(&key);
|
||
}
|
||
}
|
||
out
|
||
}
|
||
}
|
||
|
||
/// A release still open in the window: the files of one upload of its key (one after another, as they finished).
|
||
struct OpenRelease {
|
||
upload: usize,
|
||
files: Vec<Vec<Seg>>,
|
||
first: Option<i64>,
|
||
last: Option<i64>,
|
||
touched: u64,
|
||
}
|
||
|
||
impl OpenRelease {
|
||
/// How far a file's posting span is from the release's (0 when either is undated).
|
||
fn distance(&self, first: Option<i64>, last: Option<i64>) -> i64 {
|
||
match (first, last, self.first, self.last) {
|
||
(Some(ff), Some(fl), Some(rf), Some(rl)) => 0.max(rf - fl).max(ff - rl),
|
||
_ => 0,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// An album still open: the closed singles of one poster and name template (ALBUM_MAX_FILES at most), with their
|
||
/// release keys, so a lone member is written as the release it was.
|
||
struct OpenAlbum {
|
||
upload: usize,
|
||
members: Vec<(String, OpenRelease)>,
|
||
first: Option<i64>,
|
||
last: Option<i64>,
|
||
touched: u64,
|
||
}
|
||
|
||
/// The template a file name joins an album by: every digit run as "#" ("1481439079025_014.jpg.data" →
|
||
/// "#_#.jpg.data"); none for a name without a number, a random one, an archive, a volume or a sidecar.
|
||
pub fn album_template(name: &str) -> Option<String> {
|
||
let (stem, ext) = split_extension(name);
|
||
// A numbered extension is a volume ("acs5152s1.11"), and a stray suffix hides none of the above ("x.par2.temp").
|
||
let volume = |n: &str| names::is_par2(n) || names::is_archive_start(n) || names::is_later_volume(n);
|
||
if is_random_file(name) || names::is_sidecar(name) || volume(name) || volume(stem) || (ext.len() > 1 && ext.bytes().skip(1).all(|c| c.is_ascii_digit())) {
|
||
return None;
|
||
}
|
||
let mut out = String::with_capacity(name.len());
|
||
let mut runs = 0;
|
||
let mut in_digits = false;
|
||
for c in stem.chars() {
|
||
if c.is_ascii_digit() {
|
||
if !in_digits {
|
||
out.push('#');
|
||
runs += 1;
|
||
in_digits = true;
|
||
}
|
||
} else {
|
||
in_digits = false;
|
||
out.push(c);
|
||
}
|
||
}
|
||
out.push_str(ext);
|
||
(runs > 0).then_some(out)
|
||
}
|
||
|
||
/// A name's stem and its extension (a last ".xxx" of 1–5 letters and digits, "mp4"'s 4 no number sequence).
|
||
fn split_extension(name: &str) -> (&str, &str) {
|
||
match name.rfind('.') {
|
||
Some(dot) if dot > 0 && (1..=5).contains(&(name.len() - dot - 1)) && name[dot + 1..].bytes().all(|c| c.is_ascii_alphanumeric()) => (&name[..dot], &name[dot..]),
|
||
_ => (name, ""),
|
||
}
|
||
}
|
||
|
||
/// A name's runs of digits and of other characters, in order.
|
||
fn digit_runs(name: &str) -> Vec<(bool, &str)> {
|
||
let mut out = Vec::new();
|
||
let mut start = 0;
|
||
let b = name.as_bytes();
|
||
let mut i = 0;
|
||
while i < b.len() {
|
||
let digits = b[i].is_ascii_digit();
|
||
let mut j = i + 1;
|
||
while j < b.len() && b[j].is_ascii_digit() == digits {
|
||
j += 1;
|
||
}
|
||
out.push((digits, &name[start..j]));
|
||
start = j;
|
||
i = j;
|
||
}
|
||
out
|
||
}
|
||
|
||
/// An album's name: its members' shared name, with the numbers that vary among them as "#" and the ones they share
|
||
/// kept ("1481439079025_#.jpg.data" for one timestamp's pictures).
|
||
pub fn album_name(names: &[&str]) -> String {
|
||
let (stem, ext) = split_extension(names[0]);
|
||
let first = digit_runs(stem);
|
||
let mut varies = vec![false; first.len()];
|
||
for name in &names[1..] {
|
||
let runs = digit_runs(split_extension(name).0);
|
||
if runs.len() != first.len() {
|
||
return album_template(names[0]).unwrap_or_else(|| names[0].to_string());
|
||
}
|
||
for (i, (digits, text)) in runs.iter().enumerate() {
|
||
if *digits && first[i].1 != *text {
|
||
varies[i] = true;
|
||
}
|
||
}
|
||
}
|
||
let mut out = String::with_capacity(names[0].len());
|
||
for (i, (digits, text)) in first.iter().enumerate() {
|
||
if *digits && varies[i] {
|
||
out.push('#');
|
||
} else {
|
||
out.push_str(text);
|
||
}
|
||
}
|
||
out.push_str(ext);
|
||
out
|
||
}
|
||
|
||
pub struct RangeIndexer {
|
||
group: String,
|
||
rules: Rules,
|
||
end: i64,
|
||
priorities: FxHashMap<String, u32>,
|
||
idle: u64,
|
||
scanner: Scanner,
|
||
aggregator: Aggregator,
|
||
frontier: Frontier,
|
||
/// Per release key its open uploads, oldest first.
|
||
open: FxHashMap<String, Vec<OpenRelease>>,
|
||
/// Per album key its open albums, oldest first.
|
||
albums: FxHashMap<String, Vec<OpenAlbum>>,
|
||
/// The next upload number per key (distinct within a batch: the store's slot is the key and the upload).
|
||
uploads: FxHashMap<String, usize>,
|
||
/// Per (poster, counter total ≥ BATCH_COUNTER_MIN_TOTAL) the releases started under it and the batch of the last.
|
||
batch_totals: FxHashMap<(String, i64), (u32, u64)>,
|
||
batch: Batch,
|
||
counts: FxHashMap<(&'static str, String), (u64, u64)>,
|
||
count_order: Vec<(&'static str, String)>,
|
||
/// Posting times of this batch's segments (posted_span at take).
|
||
dates: Vec<i64>,
|
||
pending_files: usize,
|
||
}
|
||
|
||
impl RangeIndexer {
|
||
/// `resume_at`: where this worker starts (a range taken over midway); its first batch starts there. `idle`: batches
|
||
/// (chunks) after which an undated file or release is given up.
|
||
pub fn new(group: &str, _start: i64, end: i64, resume_at: i64, rules: Rules, idle: u64) -> Self {
|
||
RangeIndexer {
|
||
group: group.to_string(),
|
||
end,
|
||
priorities: rules.priorities.clone(),
|
||
rules,
|
||
idle,
|
||
scanner: Scanner::default(),
|
||
aggregator: Aggregator::default(),
|
||
frontier: Frontier::default(),
|
||
open: FxHashMap::default(),
|
||
albums: FxHashMap::default(),
|
||
uploads: FxHashMap::default(),
|
||
batch_totals: FxHashMap::default(),
|
||
batch: Batch { from: resume_at, next: resume_at, ..Default::default() },
|
||
counts: FxHashMap::default(),
|
||
count_order: Vec::new(),
|
||
dates: Vec::new(),
|
||
pending_files: 0,
|
||
}
|
||
}
|
||
|
||
/// A worker's indexer for a group it holds (REBUILD.md design 1): ranges come one after another (begin_range),
|
||
/// the window carried between them; the batch starts at the first range's resume point.
|
||
pub fn for_group(group: &str, rules: Rules, idle: u64) -> Self {
|
||
Self::new(group, 0, 0, 0, rules, idle)
|
||
}
|
||
|
||
/// The next range of the group: the batch starts at its resume point (after a gap too), the range's end is known.
|
||
pub fn begin_range(&mut self, next: i64, end: i64) {
|
||
self.end = end;
|
||
if self.batch.headers == 0 && self.batch.releases.is_empty() && self.batch.candidates.is_empty() {
|
||
self.batch.from = next;
|
||
self.batch.next = next;
|
||
}
|
||
}
|
||
|
||
/// The article the window stands at: the next one to index (right after a take, a batch boundary).
|
||
pub fn next(&self) -> i64 {
|
||
self.batch.next
|
||
}
|
||
|
||
/// The window's state between two batches (right after a take; the batch must be empty), serialized (bincode).
|
||
pub fn snapshot(&self) -> Result<Vec<u8>, String> {
|
||
if self.batch.headers != 0 || !self.batch.releases.is_empty() || !self.batch.candidates.is_empty() {
|
||
return Err("a snapshot is taken right after a batch, not with one open".to_string());
|
||
}
|
||
let mut table = FileTable::default();
|
||
let aggregator: Vec<(String, Vec<OpenFileSnap>)> = self
|
||
.aggregator
|
||
.open
|
||
.iter()
|
||
.map(|(key, copies)| {
|
||
(key.clone(), copies.iter().map(|c| OpenFileSnap { parts: table.segs(&c.parts), total: c.total, touched: c.touched, earliest: c.earliest, latest: c.latest }).collect())
|
||
})
|
||
.collect();
|
||
let open: Vec<(String, Vec<OpenReleaseSnap>)> = self.open.iter().map(|(key, uploads)| (key.clone(), uploads.iter().map(|u| table.release(u)).collect())).collect();
|
||
let albums: Vec<(String, Vec<OpenAlbumSnap>)> = self
|
||
.albums
|
||
.iter()
|
||
.map(|(key, list)| {
|
||
(
|
||
key.clone(),
|
||
list.iter()
|
||
.map(|a| OpenAlbumSnap { upload: a.upload, members: a.members.iter().map(|(k, u)| (k.clone(), table.release(u))).collect(), first: a.first, last: a.last, touched: a.touched })
|
||
.collect(),
|
||
)
|
||
})
|
||
.collect();
|
||
let snapshot = Snapshot {
|
||
version: SNAPSHOT_VERSION,
|
||
group: self.group.clone(),
|
||
next: self.batch.next,
|
||
frontier: Frontier { ring: self.frontier.ring.clone(), at: self.frontier.at, value: self.frontier.value, start: self.frontier.start, aside: self.frontier.aside.clone(), pending: self.frontier.pending },
|
||
files: table.into_files(),
|
||
aggregator,
|
||
aggregator_batch: self.aggregator.batch,
|
||
open,
|
||
albums,
|
||
uploads: self.uploads.iter().map(|(k, &n)| (k.clone(), n)).collect(),
|
||
batch_totals: self.batch_totals.iter().map(|((poster, total), (n, touched))| (poster.clone(), *total, *n, *touched)).collect(),
|
||
};
|
||
bincode::serialize(&snapshot).map_err(|e| e.to_string())
|
||
}
|
||
|
||
/// A worker's indexer for a group from its snapshot (the rules fresh from the lease).
|
||
pub fn restore(bytes: &[u8], rules: Rules, idle: u64) -> Result<Self, String> {
|
||
let snapshot: Snapshot = bincode::deserialize(bytes).map_err(|e| format!("snapshot: {e}"))?;
|
||
if snapshot.version != SNAPSHOT_VERSION {
|
||
return Err(format!("snapshot version {} (this build writes {SNAPSHOT_VERSION})", snapshot.version));
|
||
}
|
||
let files: Vec<Rc<FileMeta>> = snapshot.files.into_iter().map(|f| Rc::new(file_of(f))).collect();
|
||
let mut ix = Self::for_group(&snapshot.group, rules, idle);
|
||
ix.frontier = snapshot.frontier;
|
||
ix.aggregator.batch = snapshot.aggregator_batch;
|
||
for (key, copies) in snapshot.aggregator {
|
||
ix.aggregator.open.insert(
|
||
key,
|
||
copies
|
||
.into_iter()
|
||
.map(|c| {
|
||
let parts: Vec<Seg> = c.parts.into_iter().map(|s| seg_of(s, &files)).collect();
|
||
let seen = parts.iter().map(|s| s.number).collect();
|
||
OpenFile { parts, seen, total: c.total, touched: c.touched, earliest: c.earliest, latest: c.latest }
|
||
})
|
||
.collect(),
|
||
);
|
||
}
|
||
for (key, uploads) in snapshot.open {
|
||
ix.open.insert(key, uploads.into_iter().map(|u| release_of(u, &files)).collect());
|
||
}
|
||
for (key, list) in snapshot.albums {
|
||
ix.albums.insert(
|
||
key,
|
||
list.into_iter().map(|a| OpenAlbum { upload: a.upload, members: a.members.into_iter().map(|(k, u)| (k, release_of(u, &files))).collect(), first: a.first, last: a.last, touched: a.touched }).collect(),
|
||
);
|
||
}
|
||
ix.uploads = snapshot.uploads.into_iter().collect();
|
||
ix.batch_totals = snapshot.batch_totals.into_iter().map(|(poster, total, n, touched)| ((poster, total), (n, touched))).collect();
|
||
ix.batch = Batch { from: snapshot.next, next: snapshot.next, ..Default::default() };
|
||
Ok(ix)
|
||
}
|
||
|
||
pub fn add_text(&mut self, text: &str, next: i64) {
|
||
let mut segments: Vec<Seg> = Vec::new();
|
||
let mut crossposts: Vec<String> = Vec::new();
|
||
let mut obfuscated = 0u64;
|
||
let mut headers = 0u64;
|
||
{
|
||
let group = self.group.clone();
|
||
let scanner = &mut self.scanner;
|
||
let priorities = &self.priorities;
|
||
scan_overview_text(text, |n, subject, from, date, message_id, bytes, xref| {
|
||
let Some(n) = n else { return };
|
||
if message_id.is_empty() {
|
||
return;
|
||
}
|
||
headers += 1;
|
||
let owner = if xref.is_empty() { group.as_str() } else { scanner.owner(xref, &group, priorities) };
|
||
if owner != group {
|
||
crossposts.push(owner.to_string());
|
||
return;
|
||
}
|
||
match scanner.segment(n, subject, from, date, message_id, bytes, xref, &group) {
|
||
Some(s) => segments.push(s),
|
||
None => obfuscated += 1,
|
||
}
|
||
});
|
||
}
|
||
self.batch.headers += headers;
|
||
for t in segments.iter().filter_map(|s| s.posted_at) {
|
||
self.dates.push(t);
|
||
self.frontier.add(t);
|
||
}
|
||
for owner in crossposts {
|
||
self.count("crosspost", owner, 1, 0);
|
||
}
|
||
if obfuscated > 0 {
|
||
self.count("obfuscated", String::new(), obfuscated, 0);
|
||
}
|
||
self.end_chunk(segments, next);
|
||
}
|
||
|
||
fn end_chunk(&mut self, segments: Vec<Seg>, next: i64) {
|
||
let t = std::time::Instant::now();
|
||
let done = self.aggregator.add(segments);
|
||
unsafe { PROF[3] += t.elapsed().as_nanos() as u64; }
|
||
self.files(done, false);
|
||
self.frontier.update();
|
||
let t = std::time::Instant::now();
|
||
let gone = self.aggregator.evict(self.frontier.closed_before(), self.idle, false);
|
||
self.files(gone, false);
|
||
self.close(false);
|
||
unsafe { PROF[4] += t.elapsed().as_nanos() as u64; }
|
||
self.batch.next = next;
|
||
}
|
||
|
||
/// The block's end: every file and release left is written, finished when the window closed on it, else
|
||
/// provisionally and as an edge (the block merger finishes it with the neighbour block's part).
|
||
pub fn finish(&mut self) {
|
||
self.frontier.settle();
|
||
let gone = self.aggregator.evict(None, 0, true);
|
||
self.files(gone, true);
|
||
self.close(true);
|
||
self.batch.next = self.end + 1;
|
||
}
|
||
|
||
/// The next range of a group starts after a gap (edge-bench --group, the corpus holding some ranges only): a fresh
|
||
/// batch starts there, not at the previous range's end; the window carries on. Nothing when the batch has anything.
|
||
pub fn skip_to(&mut self, article: i64) {
|
||
if self.batch.headers == 0 && self.batch.releases.is_empty() && self.batch.candidates.is_empty() && self.batch.from < article {
|
||
self.batch.from = article;
|
||
self.batch.next = article;
|
||
}
|
||
}
|
||
|
||
/// Files in the releases finished since the last take (RangeIndexer.pending).
|
||
pub fn pending(&self) -> usize {
|
||
self.pending_files
|
||
}
|
||
|
||
/// Where the block is in posting time (None until FRONTIER_MIN dated articles were read).
|
||
pub fn frontier(&self) -> Option<i64> {
|
||
self.frontier.value()
|
||
}
|
||
|
||
/// Releases the window still holds (edge-bench's progress line).
|
||
pub fn open_releases(&self) -> usize {
|
||
self.open.values().map(|u| u.len()).sum::<usize>() + self.albums.values().map(|a| a.len()).sum::<usize>()
|
||
}
|
||
|
||
/// Files still short of parts, or complete and not yet in a release.
|
||
pub fn open_files(&self) -> usize {
|
||
self.aggregator.open.values().map(|f| f.len()).sum()
|
||
}
|
||
|
||
pub fn take(&mut self) -> Batch {
|
||
self.scanner.files.clear();
|
||
let next = self.batch.next;
|
||
let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() });
|
||
(out.posted_low, out.posted_high) = posted_span(std::mem::take(&mut self.dates));
|
||
for id in self.count_order.drain(..) {
|
||
let (articles, files) = self.counts.remove(&id).unwrap();
|
||
out.counts.push((id.0, id.1, articles, files));
|
||
}
|
||
self.pending_files = 0;
|
||
// Upload numbers only have to be distinct within a batch: those of keys with nothing open go.
|
||
let (open, albums) = (&self.open, &self.albums);
|
||
self.uploads.retain(|key, _| open.contains_key(key) || albums.contains_key(key));
|
||
let batch = self.aggregator.batch;
|
||
self.batch_totals.retain(|_, (_, touched)| batch - *touched < BATCH_COUNTER_BATCHES);
|
||
out
|
||
}
|
||
|
||
/// Finished files (complete, or the window closed on them) into the open release of their key and upload; `at_end`
|
||
/// when the block ends on them.
|
||
fn files(&mut self, files: Vec<Vec<Seg>>, at_end: bool) {
|
||
let batch = self.aggregator.batch;
|
||
for file in files {
|
||
let lone = file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1);
|
||
// A scrambled name only before the block's end, where the rest of a file can still be in the next block.
|
||
if lone && (is_random_file(&file[0].file.name) || (!at_end && looks_scrambled(&file[0].file.name))) {
|
||
// One article of a multi-part random file (its upload per article: ngPost's, nyuu's, and a random poster's
|
||
// per article, 160k in alt.binaries.storage; LOG.md (122)): not a release.
|
||
let tool = names::upload_tool(&file[0].message_id);
|
||
self.count("per-article", tool.to_string(), 1, 0);
|
||
continue;
|
||
}
|
||
let key = release_key_of(&file[0]);
|
||
let first = posted_of(&file);
|
||
let last = file.iter().filter_map(|s| s.posted_at).max();
|
||
let uploads = self.open.entry(key.clone()).or_default();
|
||
let near = uploads.iter_mut().filter(|u| u.distance(first, last) <= UPLOAD_GAP_MS).min_by_key(|u| u.distance(first, last));
|
||
let release = match near {
|
||
Some(u) => u,
|
||
None => {
|
||
let n = self.uploads.entry(key).or_insert(0);
|
||
let upload = *n;
|
||
*n += 1;
|
||
if let Some(total) = file[0].file.release.file_total_truthy().filter(|&t| t >= BATCH_COUNTER_MIN_TOTAL) {
|
||
let e = self.batch_totals.entry((file[0].file.poster.clone(), total)).or_insert((0, batch));
|
||
e.0 += 1;
|
||
e.1 = batch;
|
||
}
|
||
uploads.push(OpenRelease { upload, files: Vec::new(), first: None, last: None, touched: batch });
|
||
uploads.last_mut().unwrap()
|
||
}
|
||
};
|
||
if let Some(f) = first {
|
||
if release.first.is_none_or(|r| f < r) {
|
||
release.first = Some(f);
|
||
}
|
||
}
|
||
if let Some(l) = last {
|
||
if release.last.is_none_or(|r| l > r) {
|
||
release.last = Some(l);
|
||
}
|
||
}
|
||
release.touched = batch;
|
||
release.files.push(file);
|
||
}
|
||
}
|
||
|
||
/// Releases the window closed on (their last part before the frontier less WINDOW_MS; undated ones idle for
|
||
/// `idle` batches): judged and written. At the end (`all`), the rest too: provisionally, as edges.
|
||
fn close(&mut self, all: bool) {
|
||
let before = self.frontier.closed_before();
|
||
let start = self.frontier.start();
|
||
let batch = self.aggregator.batch;
|
||
let idle = self.idle;
|
||
let mut done: Vec<(String, OpenRelease, bool)> = Vec::new();
|
||
let keys: Vec<String> = self.open.keys().cloned().collect();
|
||
for key in keys {
|
||
let uploads = self.open.get_mut(&key).unwrap();
|
||
let mut i = 0;
|
||
while i < uploads.len() {
|
||
let u = &uploads[i];
|
||
let closed = match u.last {
|
||
Some(l) => before.is_some_and(|b| l < b),
|
||
None => idle > 0 && batch - u.touched >= idle,
|
||
};
|
||
if !all && !closed {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
done.push((key.clone(), uploads.remove(i), closed));
|
||
}
|
||
if uploads.is_empty() {
|
||
self.open.remove(&key);
|
||
}
|
||
}
|
||
// An edge: still open at the block's end, or posted within the window of the block's start (its other
|
||
// part may be in the neighbour block); undated ones can't be placed, so they are edges too.
|
||
let near_start = |first: Option<i64>| match (start, first) {
|
||
(Some(s), Some(f)) => f < s + WINDOW_MS,
|
||
_ => true,
|
||
};
|
||
// In posting order, so an album's members join it from its first file on (the keys' order is a hash's).
|
||
done.sort_by(|a, b| a.1.first.cmp(&b.1.first).then_with(|| a.0.cmp(&b.0)));
|
||
for (key, u, closed) in done {
|
||
// A single the window closed on waits in its album (a lone one is written as it is when that closes).
|
||
if closed {
|
||
if let Some(template) = album_candidate(&u) {
|
||
self.album(key, u, template);
|
||
continue;
|
||
}
|
||
}
|
||
let edge = !closed || near_start(u.first);
|
||
self.emit(&key, u, closed, edge);
|
||
}
|
||
// Albums: closed once every single that could join (posted within WINDOW_MS of the first) has closed too.
|
||
let mut done_albums: Vec<(String, OpenAlbum, bool)> = Vec::new();
|
||
let keys: Vec<String> = self.albums.keys().cloned().collect();
|
||
for key in keys {
|
||
let list = self.albums.get_mut(&key).unwrap();
|
||
let mut i = 0;
|
||
while i < list.len() {
|
||
let a = &list[i];
|
||
let closed = match a.first {
|
||
Some(f) => before.is_some_and(|b| f + WINDOW_MS < b),
|
||
None => idle > 0 && batch - a.touched >= idle,
|
||
};
|
||
if !all && !closed {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
done_albums.push((key.clone(), list.remove(i), closed));
|
||
}
|
||
if list.is_empty() {
|
||
self.albums.remove(&key);
|
||
}
|
||
}
|
||
for (key, mut a, closed) in done_albums {
|
||
let edge = !closed || near_start(a.first);
|
||
if a.members.len() < 2 {
|
||
let (release_key, u) = a.members.remove(0);
|
||
self.emit(&release_key, u, closed, edge);
|
||
continue;
|
||
}
|
||
let names: Vec<&str> = a.members.iter().map(|(_, u)| u.files[0][0].file.name.as_str()).collect();
|
||
let name = album_name(&names);
|
||
let files: Vec<Vec<Seg>> = a.members.into_iter().flat_map(|(_, u)| u.files).collect();
|
||
self.write(&key, a.upload, files, closed, edge, Some(name));
|
||
}
|
||
}
|
||
|
||
/// A closed single into the open album of its poster and template posted within WINDOW_MS of its first file
|
||
/// (with room), else a new one.
|
||
fn album(&mut self, release_key: String, u: OpenRelease, template: String) {
|
||
let batch = self.aggregator.batch;
|
||
let poster = &u.files[0][0].file.poster;
|
||
let poster = if names::random_poster(poster) { "" } else { poster.as_str() };
|
||
let key = sha1_hex(&format!("album\n{poster}\n{template}"));
|
||
let albums = self.albums.entry(key.clone()).or_default();
|
||
let fits = |a: &OpenAlbum| {
|
||
a.members.len() < ALBUM_MAX_FILES
|
||
&& match (a.first, u.first) {
|
||
(Some(af), Some(f)) => f >= af && f < af + WINDOW_MS,
|
||
_ => true,
|
||
}
|
||
};
|
||
let index = match albums.iter().position(fits) {
|
||
Some(i) => i,
|
||
None => {
|
||
let n = self.uploads.entry(key).or_insert(0);
|
||
let upload = *n;
|
||
*n += 1;
|
||
albums.push(OpenAlbum { upload, members: Vec::new(), first: None, last: None, touched: batch });
|
||
albums.len() - 1
|
||
}
|
||
};
|
||
let a = &mut albums[index];
|
||
if let Some(f) = u.first {
|
||
if a.first.is_none_or(|r| f < r) {
|
||
a.first = Some(f);
|
||
}
|
||
}
|
||
if let Some(l) = u.last {
|
||
if a.last.is_none_or(|r| l > r) {
|
||
a.last = Some(l);
|
||
}
|
||
}
|
||
a.touched = batch;
|
||
a.members.push((release_key, u));
|
||
}
|
||
|
||
fn emit(&mut self, key: &str, release: OpenRelease, finished: bool, edge: bool) {
|
||
self.write(key, release.upload, release.files, finished, edge, None);
|
||
}
|
||
|
||
/// A release (or an album, under its name) judged and written to the batch.
|
||
fn write(&mut self, key: &str, upload: usize, list: Vec<Vec<Seg>>, finished: bool, edge: bool, album: Option<String>) {
|
||
let judged: Vec<JudgedFile> = list
|
||
.iter()
|
||
.map(|f| JudgedFile {
|
||
name: &f[0].file.name,
|
||
total_parts: f[0].file.total_parts,
|
||
groups: &f[0].file.groups,
|
||
first_message_id: &f[0].message_id,
|
||
poster: &f[0].file.poster,
|
||
segments: f.len() as u32,
|
||
bytes: f.iter().map(|s| s.bytes.max(0) as u64).sum(),
|
||
})
|
||
.collect();
|
||
// Settled for the single-archive rules: the window closed on it (a par2 can still join an open one). The
|
||
// start edge (posted within the window of the block's start) doesn't count: a par2 comes after its rar, so
|
||
// one in the previous block is rare (0 par2 siblings of 3,000 catalog hits, LOG.md (125)), and 7% of the
|
||
// catalog's releases are start edges, which would escape the rule.
|
||
match names::judge(key, &judged, finished, &self.rules) {
|
||
Verdict::Keep => {
|
||
let candidates = candidates_of(key, &list, upload);
|
||
let mut chunk = chunk_of(key, &list, upload);
|
||
chunk.finished = finished;
|
||
chunk.edge = edge;
|
||
let album_none = album.is_none();
|
||
if let Some(name) = album {
|
||
chunk.name = name;
|
||
chunk.source = "album";
|
||
chunk.file_total = None;
|
||
}
|
||
if !chunk.readable {
|
||
// Unnamed: no chunk (its segments are read again once a probe names it), the tool for its signature.
|
||
chunk.tool = Some(names::upload_tool(&list[0][0].message_id));
|
||
chunk.payload = encode_chunk(&[]);
|
||
}
|
||
if chunk.readable {
|
||
self.pending_files += chunk.files;
|
||
}
|
||
if let Some(total) = chunk.file_total.filter(|&t| album_none && t >= BATCH_COUNTER_MIN_TOTAL && (chunk.files as i64) < t) {
|
||
// A batch counter: no file total (the release is as complete as its files are).
|
||
if self.batch_totals.get(&(chunk.poster.clone(), total)).is_some_and(|(n, _)| *n >= BATCH_COUNTER_RELEASES) {
|
||
chunk.file_total = None;
|
||
}
|
||
}
|
||
self.batch.releases.push(chunk);
|
||
self.batch.candidates.extend(candidates);
|
||
}
|
||
Verdict::Drop { rule, key: rule_key, sample } => {
|
||
let segments: usize = list.iter().map(|f| f.len()).sum();
|
||
self.count(rule, rule_key.clone(), segments as u64, list.len() as u64);
|
||
if sample {
|
||
self.batch.samples.push((chunk_of(key, &list, upload), rule, rule_key));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
fn count(&mut self, rule: &'static str, key: String, articles: u64, files: u64) {
|
||
match self.counts.get_mut(&(rule, key.clone())) {
|
||
Some(c) => {
|
||
c.0 += articles;
|
||
c.1 += files;
|
||
}
|
||
None => {
|
||
self.count_order.push((rule, key.clone()));
|
||
self.counts.insert((rule, key), (articles, files));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// A closed release that may be an album's member: one file, named by its stem or its post (no release name in the
|
||
/// subject), with a number in its name (album_template).
|
||
fn album_candidate(u: &OpenRelease) -> Option<String> {
|
||
if u.files.len() != 1 {
|
||
return None;
|
||
}
|
||
let meta = &u.files[0][0].file;
|
||
if !matches!(meta.release.source, "stem" | "post") {
|
||
return None;
|
||
}
|
||
album_template(&meta.name)
|
||
}
|
||
|
||
fn release_key_of(s: &Seg) -> String {
|
||
// A named file always has a release (releaseIdentity isn't null unless weak).
|
||
s.file.release.key.clone()
|
||
}
|
||
|
||
fn posted_of(segments: &[Seg]) -> Option<i64> {
|
||
segments.iter().filter_map(|s| s.posted_at).min()
|
||
}
|
||
|
||
fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
|
||
let first = &files[0][0].file;
|
||
let chunk_files: Vec<ChunkFile> = files
|
||
.iter()
|
||
.map(|f| {
|
||
let m = &f[0].file;
|
||
ChunkFile { name: &m.name, poster: &m.poster, subject: &m.subject, posted_at: posted_of(f), groups: &m.groups, segments: f.iter().map(|s| (s.number, s.message_id.as_str(), s.bytes)).collect() }
|
||
})
|
||
.collect();
|
||
let mut segments = 0;
|
||
let mut bytes = 0;
|
||
let mut first_article = i64::MAX;
|
||
let mut last_article = 0;
|
||
let mut posted_at: Option<i64> = None;
|
||
let mut posted_last: Option<i64> = None;
|
||
for f in files {
|
||
for s in f {
|
||
if let Some(t) = s.posted_at {
|
||
if posted_last.is_none_or(|p| t > p) {
|
||
posted_last = Some(t);
|
||
}
|
||
}
|
||
segments += 1;
|
||
bytes += s.bytes;
|
||
first_article = first_article.min(s.article);
|
||
last_article = last_article.max(s.article);
|
||
}
|
||
if let Some(t) = posted_of(f) {
|
||
if posted_at.is_none_or(|p| t < p) {
|
||
posted_at = Some(t);
|
||
}
|
||
}
|
||
}
|
||
ReleaseChunk {
|
||
key: key.to_string(),
|
||
upload,
|
||
name: first.release.name.clone(),
|
||
source: first.release.source,
|
||
file_total: files.iter().find_map(|f| f[0].file.release.file_total_truthy()),
|
||
poster: first.poster.clone(),
|
||
groups: first.groups.clone(),
|
||
posted_at,
|
||
posted_last,
|
||
files: files.len(),
|
||
segments,
|
||
bytes,
|
||
first_article,
|
||
last_article,
|
||
readable: chunk_files.iter().any(|f| !is_random_file(f.name)),
|
||
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
||
finished: true,
|
||
edge: false,
|
||
partial: files.iter().any(|f| (f.len() as u32) < f[0].file.total_parts),
|
||
tool: None,
|
||
payload: encode_chunk(&chunk_files),
|
||
subjects: chunk_files.iter().take(SUBJECTS).map(|f| f.subject.to_string()).collect(),
|
||
}
|
||
}
|
||
|
||
fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate> {
|
||
let mut out = Vec::new();
|
||
let mut tail: Option<(i64, &Vec<Seg>)> = None;
|
||
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
|
||
for f in files {
|
||
let name = &f[0].file.name;
|
||
let volume = names::seven_zip_volume(name);
|
||
if let Some(volume) = volume.filter(|&v| v > 1) {
|
||
if tail.is_none_or(|(v, _)| volume > v) {
|
||
tail = Some((volume, f));
|
||
}
|
||
continue;
|
||
}
|
||
// An unsplit 7z of more than one segment: its listing is in its last one.
|
||
if volume == Some(0) && f.len() > 1 && tail.is_none() {
|
||
tail = Some((0, f));
|
||
}
|
||
if names::is_archive_start(name) {
|
||
out.push(c("archive", &f[0]));
|
||
} else if names::is_par2(name) && !names::is_later_volume(name) {
|
||
out.push(c("par2", &f[0]));
|
||
}
|
||
}
|
||
if let Some((_, f)) = tail {
|
||
out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() });
|
||
}
|
||
if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) {
|
||
out.push(c("lone", &files[0][0]));
|
||
}
|
||
out.truncate(4);
|
||
out
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
const GROUP: &str = "alt.binaries.test";
|
||
|
||
fn line(n: i64, subject: &str, minutes: i64) -> String {
|
||
let date = chrono::DateTime::from_timestamp_millis(1_791_194_400_000 + minutes * 60_000).unwrap().format("%a, %d %b %Y %H:%M:%S +0000");
|
||
format!("{n}\t{subject}\tp <p@x>\t{date}\t<u{n}@test>\t\t1000\t10\tXref: news.example {GROUP}:{n}")
|
||
}
|
||
|
||
#[test]
|
||
fn uploads_split_as_in_the_typescript() {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
||
let mut lines: Vec<String> = (0..5).map(|i| line(100 + i, &format!(r#"Big.Release [1/2] - "Big.Release.mkv" yEnc ({}/5)"#, i + 1), i * 60)).collect();
|
||
lines.push(line(200, r#"Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)"#, 245));
|
||
lines.extend((0..5).map(|i| line(300 + i, &format!(r#"Big.Release [1/2] - "Big.Release.mkv" yEnc ({}/5)"#, i + 1), 1440 + i)));
|
||
ix.add_text(&lines.join("\r\n"), 10_001);
|
||
ix.finish();
|
||
let batch = ix.take();
|
||
let got: Vec<(usize, usize)> = batch.releases.iter().map(|r| (r.upload, r.files)).collect();
|
||
assert_eq!(got, vec![(0, 2), (1, 1)]);
|
||
assert_eq!(batch.releases[0].posted_last.unwrap() - batch.releases[0].posted_at.unwrap(), 245 * 60_000);
|
||
assert_eq!(batch.releases[0].key, batch.releases[1].key);
|
||
// A day of posting in all: the window never closed on them, so at the end they are provisional edges.
|
||
assert!(batch.releases.iter().all(|r| !r.finished && r.edge && !r.partial));
|
||
}
|
||
|
||
/// `n` single-part filler files posted `minutes` apart from `start`: dated articles that move the frontier.
|
||
fn fillers(from: i64, n: i64, start: i64, minutes: i64) -> Vec<String> {
|
||
(0..n).map(|i| line(from + i, &format!(r#"Filler.{} [1/1] - "filler.{}.jpg" yEnc (1/1)"#, from + i, from + i), start + i * minutes)).collect()
|
||
}
|
||
|
||
#[test]
|
||
fn the_window_finishes_releases_and_keeps_incomplete_files() {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
let mut ix = RangeIndexer::new(GROUP, 1, 1_000_000, 1, Rules::defaults(priorities), 5);
|
||
let day = 24 * 60;
|
||
// Chunk 1: a release at the block's start (two of three parts of its file), and 100 dated articles after it.
|
||
let mut lines = vec![
|
||
line(10, r#"Early.Show [1/1] - "Early.Show.mkv" yEnc (1/3)"#, 0),
|
||
line(11, r#"Early.Show [1/1] - "Early.Show.mkv" yEnc (2/3)"#, 1),
|
||
];
|
||
lines.extend(fillers(100, 100, 2, 1));
|
||
ix.add_text(&lines.join("
|
||
"), 1000);
|
||
assert!(ix.frontier().is_some());
|
||
let b1 = ix.take();
|
||
assert!(b1.releases.iter().all(|r| r.name.starts_with("Filler")));
|
||
// Chunk 2, three days on (more articles than before, so the median moves there): the window closed on the
|
||
// early show (its file still short a part) and on the first fillers.
|
||
let mut lines = vec![line(2000, r#"Later.Show [1/1] - "Later.Show.mkv" yEnc (1/1)"#, 3 * day)];
|
||
lines.extend(fillers(2100, 300, 3 * day + 1, 1));
|
||
ix.add_text(&lines.join("
|
||
"), 3000);
|
||
let b2 = ix.take();
|
||
let early = b2.releases.iter().find(|r| r.name == "Early Show").expect("the early show is written");
|
||
assert!(early.finished && early.edge && early.partial, "finished {} edge {} partial {}", early.finished, early.edge, early.partial);
|
||
assert_eq!((early.files, early.segments), (1, 2));
|
||
assert!(!b2.releases.iter().any(|r| r.name == "Later Show"));
|
||
// Chunk 3, three more days: the later show is finished, and not an edge (a day past the block's start).
|
||
ix.add_text(&fillers(4000, 500, 6 * day, 1).join("
|
||
"), 5000);
|
||
let b3 = ix.take();
|
||
let later = b3.releases.iter().find(|r| r.name == "Later Show").expect("the later show is written");
|
||
assert!(later.finished && !later.edge && !later.partial);
|
||
// The end: the last fillers are still inside the window, so they are provisional edges.
|
||
ix.finish();
|
||
let b4 = ix.take();
|
||
assert_eq!(b4.releases.len(), 500);
|
||
assert!(b4.releases.iter().all(|r| !r.finished && r.edge));
|
||
assert_eq!(b4.next, 1_000_001);
|
||
}
|
||
|
||
#[test]
|
||
fn singles_of_one_template_become_an_album() {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
let mut ix = RangeIndexer::new(GROUP, 1, 1_000_000, 1, Rules::defaults(priorities), 5);
|
||
let day = 24 * 60;
|
||
// A dump's pictures of one timestamp (a stem each: ".data" is no image), two scans, a lone document, and dated fillers.
|
||
let mut lines: Vec<String> = (1..=14).map(|i| line(1000 + i, &format!(r#"<IMGBANK> [1/1] "1481439079025_{i:03}.jpg.data" yEnc (1/1)"#), i)).collect();
|
||
lines.push(line(2001, r#"[1/1] "Scan_001.pdf" yEnc (1/1)"#, 20));
|
||
lines.push(line(2002, r#"[1/1] "Scan_002.pdf" yEnc (1/1)"#, 21));
|
||
lines.push(line(2003, r#"[1/1] "Lonely_007.pdf" yEnc (1/1)"#, 22));
|
||
lines.extend(fillers(3000, 100, 23, 1));
|
||
ix.add_text(&lines.join("\n"), 4000);
|
||
let b1 = ix.take();
|
||
assert!(b1.releases.iter().all(|r| r.name.starts_with("Filler")));
|
||
// Three days on: the window closed on the singles, and on their albums.
|
||
ix.add_text(&fillers(5000, 300, 3 * day, 1).join("\n"), 6000);
|
||
let b2 = ix.take();
|
||
let mut albums: Vec<(&str, &str, usize, bool)> = b2.releases.iter().filter(|r| !r.name.starts_with("Filler")).map(|r| (r.name.as_str(), r.source, r.files, r.finished)).collect();
|
||
albums.sort();
|
||
assert_eq!(albums, vec![("1481439079025_#.jpg.data", "album", 14, true), ("Lonely_007", "stem", 1, true), ("Scan_#.pdf", "album", 2, true)]);
|
||
let album = b2.releases.iter().find(|r| r.source == "album" && r.files == 14).unwrap();
|
||
// Posted within the window of the block's start: an edge, like any release there.
|
||
assert!(album.file_total.is_none() && album.edge && !album.partial && album.readable);
|
||
assert_eq!(album.segments, 14);
|
||
assert_eq!(b2.candidates.len(), 0);
|
||
assert!(ix.albums.is_empty());
|
||
}
|
||
|
||
#[test]
|
||
fn an_unnamed_release_keeps_no_chunk_but_its_tool_and_probe_target() {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
||
let lines: Vec<String> = (0..4).map(|i| line(100 + i, &format!(r#"[1/1] - "q3B9OrpKiCxUnzgtUGnp.part01.rar" yEnc ({}/4)"#, i + 1), i)).collect();
|
||
ix.add_text(&lines.join("\n"), 10_001);
|
||
ix.finish();
|
||
let batch = ix.take();
|
||
assert_eq!(batch.releases.len(), 1);
|
||
let r = &batch.releases[0];
|
||
assert!(!r.readable && r.tool == Some("other") && r.files == 1 && r.segments == 4);
|
||
assert_eq!(r.payload, encode_chunk(&[]));
|
||
assert_eq!(batch.candidates.iter().map(|c| c.kind).collect::<Vec<_>>(), vec!["archive"]);
|
||
assert_eq!(ix.pending(), 0);
|
||
}
|
||
|
||
#[test]
|
||
fn album_templates_and_names() {
|
||
assert_eq!(album_template("1481439079025_014.jpg.data").as_deref(), Some("#_#.jpg.data"));
|
||
assert_eq!(album_template("ActiCalc 1.06.rar"), None);
|
||
assert_eq!(album_template("Scan_001.vol00+01.par2"), None);
|
||
assert_eq!(album_template("Scan_001.nfo"), None);
|
||
assert_eq!(album_template("NoNumber.mp4"), None);
|
||
assert_eq!(album_template("acs5152s1.11"), None);
|
||
assert_eq!(album_template("Broforce.vol00+01.par2.temp"), None);
|
||
assert_eq!(album_template("Starship For Rent 3 (2024).epub").as_deref(), Some("Starship For Rent # (#).epub"));
|
||
assert_eq!(album_template("c3453cdd2a3645a8f3cf3d22319abe63"), None);
|
||
assert_eq!(album_name(&["1481439079025_001.jpg.data", "1481439079025_014.jpg.data"]), "1481439079025_#.jpg.data");
|
||
assert_eq!(album_name(&["1481439079025_001.jpg.data", "1481439080000_001.jpg.data"]), "#_001.jpg.data");
|
||
assert_eq!(album_name(&["Scan_001.pdf", "Scan_002.pdf", "Scan_003.pdf"]), "Scan_#.pdf");
|
||
}
|
||
|
||
#[test]
|
||
fn a_batch_counter_is_no_file_total() {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
let mut ix = RangeIndexer::new(GROUP, 1, 1_000_000, 1, Rules::defaults(priorities), 5);
|
||
let day = 24 * 60;
|
||
// 25 different programs of one poster under one running counter, and one upload with a counter of its own.
|
||
let mut lines: Vec<String> = (0..25).map(|i| line(100 + i, &format!(r#"[{}/68067] - "Program.Number.{i}.v1.0.rar" yEnc (1/1)"#, 1000 + i), i)).collect();
|
||
lines.push(line(200, r#"[1/120] - "Real.Upload.part001.rar" yEnc (1/1)"#, 30));
|
||
lines.extend(fillers(300, 100, 31, 1));
|
||
ix.add_text(&lines.join("\n"), 1000);
|
||
ix.add_text(&fillers(2000, 300, 3 * day, 1).join("\n"), 3000);
|
||
let batch = ix.take();
|
||
let programs: Vec<&ReleaseChunk> = batch.releases.iter().filter(|r| r.name.starts_with("Program")).collect();
|
||
assert_eq!(programs.len(), 25);
|
||
assert!(programs.iter().all(|r| r.file_total.is_none() && r.files == 1));
|
||
let real = batch.releases.iter().find(|r| r.name.starts_with("Real")).unwrap();
|
||
assert_eq!(real.file_total, Some(120));
|
||
}
|
||
|
||
#[test]
|
||
fn a_snapshot_restores_the_window_between_two_ranges() {
|
||
let rules = || {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
Rules::defaults(priorities)
|
||
};
|
||
let day = 24 * 60;
|
||
// Range 1: a release whose last file comes in range 2, a day later; an album's first member; fillers.
|
||
let mut first = vec![
|
||
line(10, r#"Two.Ranges [1/2] - "Two.Ranges.part1.rar" yEnc (1/1)"#, 0),
|
||
line(11, r#"[1/1] "Scan_001.pdf" yEnc (1/1)"#, 1),
|
||
];
|
||
first.extend(fillers(100, 100, 2, 1));
|
||
// The second file within the upload gap of the first (3 h), the album's second member within its window.
|
||
let mut second = vec![
|
||
line(1000, r#"Two.Ranges [2/2] - "Two.Ranges.part2.rar" yEnc (1/1)"#, 100),
|
||
line(1001, r#"[1/1] "Scan_002.pdf" yEnc (1/1)"#, 101),
|
||
];
|
||
second.extend(fillers(2000, 300, 3 * day, 1));
|
||
let summary = |batches: Vec<Batch>| {
|
||
let mut got: Vec<(String, &str, usize, bool)> = batches.iter().flat_map(|b| b.releases.iter()).filter(|r| !r.name.starts_with("Filler")).map(|r| (r.name.clone(), r.source, r.files, r.finished)).collect();
|
||
got.sort();
|
||
got
|
||
};
|
||
// In one go.
|
||
let mut whole = RangeIndexer::for_group(GROUP, rules(), 5);
|
||
whole.begin_range(1, 999);
|
||
whole.add_text(&first.join("\n"), 1000);
|
||
let w1 = whole.take();
|
||
whole.begin_range(1000, 1_000_000);
|
||
whole.add_text(&second.join("\n"), 1_000_001);
|
||
let w2 = whole.take();
|
||
// With a snapshot between the ranges, restored into a fresh indexer.
|
||
let mut before = RangeIndexer::for_group(GROUP, rules(), 5);
|
||
before.begin_range(1, 999);
|
||
before.add_text(&first.join("\n"), 1000);
|
||
let s1 = before.take();
|
||
assert_eq!(before.next(), 1000);
|
||
let bytes = before.snapshot().unwrap();
|
||
assert!(bytes.len() > 1000);
|
||
let mut after = RangeIndexer::restore(&bytes, rules(), 5).unwrap();
|
||
assert_eq!(after.next(), 1000);
|
||
assert_eq!(after.open_releases(), before.open_releases());
|
||
assert_eq!(after.open_files(), before.open_files());
|
||
after.begin_range(1000, 1_000_000);
|
||
after.add_text(&second.join("\n"), 1_000_001);
|
||
let s2 = after.take();
|
||
assert_eq!(summary(vec![s1, s2]), summary(vec![w1, w2]));
|
||
let got = summary(vec![after.take()]);
|
||
assert!(got.is_empty());
|
||
// The two-range release is one, finished, with both files; the album joined its second member.
|
||
let mut a2 = RangeIndexer::for_group(GROUP, rules(), 5);
|
||
a2.begin_range(1, 999);
|
||
a2.add_text(&first.join("\n"), 1000);
|
||
let _ = a2.take();
|
||
let restored = RangeIndexer::restore(&a2.snapshot().unwrap(), rules(), 5).unwrap();
|
||
let _ = restored;
|
||
let s2b = {
|
||
let mut r = RangeIndexer::restore(&bytes, rules(), 5).unwrap();
|
||
r.begin_range(1000, 1_000_000);
|
||
r.add_text(&second.join("\n"), 1_000_001);
|
||
r.take()
|
||
};
|
||
let two = s2b.releases.iter().find(|r| r.name == "Two Ranges").expect("the two-range release");
|
||
assert!(two.finished && two.files == 2);
|
||
assert!(s2b.releases.iter().any(|r| r.source == "album" && r.files == 2));
|
||
// A snapshot with a batch open is refused; one of another version too.
|
||
let mut open = RangeIndexer::for_group(GROUP, rules(), 5);
|
||
open.begin_range(1, 999);
|
||
open.add_text(&first.join("\n"), 1000);
|
||
assert!(open.snapshot().is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn the_frontier_ignores_implausible_dates_and_follows_a_pause() {
|
||
let hour = 3_600_000i64;
|
||
let mut f = Frontier::default();
|
||
for i in 0..200 {
|
||
f.add(i * hour / 10);
|
||
}
|
||
f.update();
|
||
let settled = f.value().unwrap();
|
||
assert!((settled - 10 * hour).abs() <= hour);
|
||
assert_eq!(f.start(), Some(settled));
|
||
// A bogus year, and one a year back: set aside.
|
||
for _ in 0..100 {
|
||
f.add(4_000_000_000_000);
|
||
f.add(-400 * 24 * hour);
|
||
}
|
||
f.update();
|
||
assert_eq!(f.value(), Some(settled));
|
||
// The group paused for a month: FRONTIER_ARTICLES agreeing dates move the frontier there.
|
||
let later = 30 * 24 * hour;
|
||
for i in 0..FRONTIER_ARTICLES as i64 {
|
||
f.add(later + i * 1000);
|
||
}
|
||
f.update();
|
||
assert!(f.value().unwrap() >= later);
|
||
assert_eq!(f.start(), Some(settled));
|
||
}
|
||
|
||
#[test]
|
||
fn archive_candidates_as_in_the_typescript() {
|
||
let mut priorities = FxHashMap::default();
|
||
priorities.insert(GROUP.to_string(), 1);
|
||
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
||
let file = |at: i64, release: &str, name: &str, n: usize, parts: i64| -> Vec<String> {
|
||
(0..parts).map(|i| line(at + i, &format!(r#"{release} [{n}/3] - "{name}" yEnc ({}/{parts})"#, i + 1), 0)).collect()
|
||
};
|
||
let mut lines = Vec::new();
|
||
// A split 7z: its start, and the end of its highest volume.
|
||
lines.extend(file(100, "Split.Seven", "Split.Seven.7z.001", 1, 2));
|
||
lines.extend(file(110, "Split.Seven", "Split.Seven.7z.003", 3, 2));
|
||
lines.extend(file(120, "Split.Seven", "Split.Seven.7z.002", 2, 2));
|
||
// An unsplit 7z of 3 segments: its start and its end.
|
||
lines.extend(file(200, "Whole.Seven", "Whole.Seven.7z", 1, 3));
|
||
// A ZIP: its start.
|
||
lines.extend(file(300, "Some.Zip", "Some.Zip.zip", 1, 2));
|
||
ix.add_text(&lines.join("\r\n"), 10_001);
|
||
ix.finish();
|
||
let batch = ix.take();
|
||
let mut got: Vec<(&str, &str, &str)> = batch.candidates.iter().map(|c| (c.kind, c.file.as_str(), c.message_id.as_str())).collect();
|
||
got.sort();
|
||
assert_eq!(
|
||
got,
|
||
vec![
|
||
("7z-tail", "Split.Seven.7z.003", "u111@test"),
|
||
("7z-tail", "Whole.Seven.7z", "u202@test"),
|
||
("archive", "Some.Zip.zip", "u300@test"),
|
||
("archive", "Split.Seven.7z.001", "u100@test"),
|
||
("archive", "Whole.Seven.7z", "u200@test"),
|
||
]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn copies_and_fills_as_in_the_typescript() {
|
||
let hour = 3_600_000i64;
|
||
let meta = Rc::new(FileMeta {
|
||
file_key: "a".into(),
|
||
name: "a.bin".into(),
|
||
poster: "p".into(),
|
||
subject: "s".into(),
|
||
total_parts: 3,
|
||
groups: vec![],
|
||
release: names::release_identity("p", "s", "a.bin"),
|
||
});
|
||
let at = |number: u32, h: f64, id: &str| Seg { file: meta.clone(), number, message_id: id.into(), bytes: 100, article: 0, posted_at: Some((h * hour as f64) as i64) };
|
||
let mut agg = Aggregator::default();
|
||
assert!(agg.add(vec![at(1, 10.0, "a1"), at(2, 10.0, "a2"), at(1, 10.1, "resend")]).is_empty());
|
||
assert!(agg.add(vec![at(1, 34.0, "b1"), at(2, 34.0, "b2")]).is_empty());
|
||
assert_eq!(agg.open.get("a").unwrap().len(), 2);
|
||
let second = agg.add(vec![at(3, 34.2, "b3")]);
|
||
assert_eq!(second[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["b1", "b2", "b3"]);
|
||
let first = agg.add(vec![at(3, 58.0, "a3")]);
|
||
assert_eq!(first[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["a1", "a2", "a3"]);
|
||
assert!(agg.open.is_empty());
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod unquoted_tests {
|
||
use super::unquoted_name;
|
||
|
||
#[test]
|
||
fn file_names_of_subjects_without_a_quoted_one() {
|
||
let cases = [
|
||
("DVDFab 8.2.0.5.rar ", Some("DVDFab 8.2.0.5.rar")),
|
||
("q3B9OrpKiCxUnzgtUGnp.part023.rar ", Some("q3B9OrpKiCxUnzgtUGnp.part023.rar")),
|
||
("New_Super_Mario_Bros._[Keygen].exe ", Some("New_Super_Mario_Bros._[Keygen].exe")),
|
||
("Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE ", Some("Emily_18_-_Bed.part01.rar")),
|
||
("[33/117] 674a37548b09.dat ", Some("674a37548b09.dat")),
|
||
("A Strange Light from the East - Eastern Influenced Western Music (1965-72) - CD2 - 07 - Rascals - Sattva.mp3 ", Some("07 - Rascals - Sattva.mp3")),
|
||
("pal! [Mivina high(1).wmv\" yEnc] ", Some("Mivina high(1).wmv")),
|
||
("[03/21 Geroellheimer - S03E16.part01.rar\" yEnc ", Some("Geroellheimer - S03E16.part01.rar")),
|
||
("<Group> FP -> {EXL01~P~004} -> Alexia (3) -> VF -> [0001 of 0076] -> Exclusive-tgirls_000000-lg.jpg ", Some("Exclusive-tgirls_000000-lg.jpg")),
|
||
("Ballers.2014.S01E01.HDTV.x264-KILLERS[ettv] 15_07_2015.rar ", Some("Ballers.2014.S01E01.HDTV.x264-KILLERS[ettv] 15_07_2015.rar")),
|
||
("Computer Bild - Ausgabe 152015 - 15_07_2015.rar ", Some("Computer Bild - Ausgabe 152015 - 15_07_2015.rar")),
|
||
("<abm> <Duke> here's a little thanksgiving present: AB: part2 0573-1002 043.jpg ", Some("043.jpg")),
|
||
("xlady post You, Me And Dupree www.x2l.nl [001/106] 'You, Me And Dupree.nfo'", Some("You, Me And Dupree.nfo")),
|
||
("[01/77][LIBRATEAM_M2219009.par2] Onbeperkt downloaden met een snelheid van 1,22 MB p.s. en dat voor maar 7,95 p.m. www.libranews.nl", Some("LIBRATEAM_M2219009.par2")),
|
||
("patensas post -^-[Mnozil Brass - Seven.nzb]-^-www.usenet2day.cc -^-[000/107]-^-www.libranews.nl-^- yEnc", Some("Mnozil Brass - Seven.nzb")),
|
||
("[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc ", None),
|
||
("Re: Will somebody answer the question? ", None),
|
||
];
|
||
for (text, want) in cases {
|
||
assert_eq!(unquoted_name(text).as_deref(), want, "{text}");
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod beside_quotes_tests {
|
||
use super::name_beside_quotes;
|
||
|
||
#[test]
|
||
fn a_name_beside_a_quoted_word() {
|
||
assert_eq!(name_beside_quotes("Official", "My \"Official\" remaining set's of Inna, Please help up to #69 if you can 025.jpg [25/74] - yEnc ").as_deref(), Some("025.jpg"));
|
||
assert_eq!(name_beside_quotes("Dr. Lust", "\"Dr. Lust\" - Lily Marlene, Vanessa Del Rio, Scorpio? - yEnc - [12/21] - Ed_DRL_VDR_LM.part09.rar ").as_deref(), Some("Ed_DRL_VDR_LM.part09.rar"));
|
||
assert_eq!(name_beside_quotes("1f0554e5fec14b8b05ca", "77cd6d03d19281a775f5bdc5 [2/8] - \"1f0554e5fec14b8b05ca\" yEnc ").as_deref(), None);
|
||
assert_eq!(name_beside_quotes("x.rar", "\"x.rar\" other.zip ").as_deref(), None);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod posted_span_tests {
|
||
use super::posted_span;
|
||
|
||
#[test]
|
||
fn without_the_earliest_and_latest_percent() {
|
||
assert_eq!(posted_span(vec![]), (None, None));
|
||
let mut dates: Vec<i64> = (0..200).map(|i| 1_000_000 + i).collect();
|
||
dates.push(5);
|
||
dates.push(9_999_999_999);
|
||
// 202 dates: 2 trimmed at each end, so the two wild ones go (postedSpan).
|
||
assert_eq!(posted_span(dates), (Some(1_000_001), Some(1_000_198)));
|
||
}
|
||
}
|