Daan MeijerandClaude Opus 5.5
229189f35c
- "Ballers.2014.S01E01.HDTV.x264-KILLERS[ettv] 15_07_2015.rar", "Computer Bild - Ausgabe 152015 - 15_07_2015.rar": the cut after "] " or " - " left "15_07_2015.rar" for every item of a Razor1911 dump (1,335 files for 12); a date alone after the cut means the whole text is the name. - "MDG005b - (01/90) "IMG_2307.JPG"": a counter in parentheses makes no post key, so every "IMG" set of the poster was one (609 for 90); an image under a post token is keyed by it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1018 lines
39 KiB
Rust
1018 lines
39 KiB
Rust
//! lib/edge/scan.ts (SegmentScanner, scanOverviewText), lib/files/aggregator.ts
|
|
//! and lib/edge/range_indexer.ts (RangeIndexer, chunkOf, candidatesOf), in the
|
|
//! same order of operations, so the outputs can be compared one to one.
|
|
|
|
use crate::codec::{encode_chunk, ChunkFile};
|
|
use crate::js::{self, sha1_hex};
|
|
use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict};
|
|
use rustc_hash::{FxHashMap, FxHashSet};
|
|
use std::rc::Rc;
|
|
|
|
pub static mut PROF: [u64; 8] = [0; 8];
|
|
|
|
pub struct FileMeta {
|
|
pub file_key: String,
|
|
pub name: String,
|
|
pub poster: String,
|
|
pub subject: String,
|
|
pub total_parts: u32,
|
|
pub groups: Vec<String>,
|
|
pub release: Release,
|
|
}
|
|
|
|
pub struct Seg {
|
|
pub file: Rc<FileMeta>,
|
|
pub number: u32,
|
|
pub message_id: String,
|
|
pub bytes: i64,
|
|
pub article: i64,
|
|
pub posted_at: Option<i64>,
|
|
}
|
|
|
|
pub struct ReleaseChunk {
|
|
pub key: String,
|
|
pub upload: usize,
|
|
pub name: String,
|
|
pub source: &'static str,
|
|
pub file_total: Option<i64>,
|
|
pub poster: String,
|
|
pub groups: Vec<String>,
|
|
pub posted_at: Option<i64>,
|
|
pub posted_last: Option<i64>,
|
|
pub files: usize,
|
|
pub segments: usize,
|
|
pub bytes: i64,
|
|
pub first_article: i64,
|
|
pub last_article: i64,
|
|
pub readable: bool,
|
|
pub has_par2: bool,
|
|
pub payload: Vec<u8>,
|
|
/// The first files' subjects (at most SUBJECTS): edge-bench's --releases dump for the grouping sweeps.
|
|
pub subjects: Vec<String>,
|
|
}
|
|
|
|
pub const SUBJECTS: usize = 3;
|
|
|
|
pub struct Candidate {
|
|
pub release_key: String,
|
|
pub upload: usize,
|
|
pub kind: &'static str,
|
|
pub message_id: String,
|
|
pub file: String,
|
|
}
|
|
|
|
pub struct Fragment {
|
|
pub file_key: String,
|
|
pub name: String,
|
|
pub poster: String,
|
|
pub subject: String,
|
|
pub total_parts: u32,
|
|
pub groups: Vec<String>,
|
|
pub release: Release,
|
|
pub posted_at: Option<i64>,
|
|
pub segments: Vec<(u32, String, i64, i64)>,
|
|
}
|
|
|
|
#[derive(Default)]
|
|
pub struct Batch {
|
|
pub from: i64,
|
|
pub next: i64,
|
|
pub headers: u64,
|
|
pub releases: Vec<ReleaseChunk>,
|
|
pub samples: Vec<(ReleaseChunk, &'static str, String)>,
|
|
pub candidates: Vec<Candidate>,
|
|
/// (rule, key) → (articles, files)
|
|
pub counts: Vec<(&'static str, String, u64, u64)>,
|
|
pub fragments: Vec<Fragment>,
|
|
}
|
|
|
|
struct Counter {
|
|
index: usize,
|
|
length: usize,
|
|
segment: i64,
|
|
total: i64,
|
|
}
|
|
|
|
/// The last "(k/N)" after `after` (anywhere when None: JS -1), scanning back from the end (findCounter).
|
|
fn find_counter(subject: &[u8], after: Option<usize>) -> Option<Counter> {
|
|
let mut open = subject.len();
|
|
loop {
|
|
// lastIndexOf('(', open - 1): at most open - 1.
|
|
if open == 0 {
|
|
return None;
|
|
}
|
|
open = subject[..open].iter().rposition(|&c| c == b'(')?;
|
|
if after.is_some_and(|a| open <= a) {
|
|
return None;
|
|
}
|
|
let mut i = open + 1;
|
|
let mut segment: i64 = 0;
|
|
let mut digits = 0;
|
|
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
|
segment = segment * 10 + (c - b'0') as i64;
|
|
digits += 1;
|
|
i += 1;
|
|
}
|
|
if digits == 0 || subject.get(i) != Some(&b'/') {
|
|
continue;
|
|
}
|
|
i += 1;
|
|
let mut total: i64 = 0;
|
|
digits = 0;
|
|
while let Some(c) = subject.get(i).filter(|c| c.is_ascii_digit()) {
|
|
total = total * 10 + (c - b'0') as i64;
|
|
digits += 1;
|
|
i += 1;
|
|
}
|
|
if digits == 0 || subject.get(i) != Some(&b')') {
|
|
continue;
|
|
}
|
|
return Some(Counter { index: open, length: i + 1 - open, segment, total });
|
|
}
|
|
}
|
|
|
|
fn xref_groups(xref: &str, group: &str) -> Vec<String> {
|
|
let mut groups = Vec::new();
|
|
for token in js::split_spaces(js::trim(xref)).into_iter().skip(1) {
|
|
if let Some(colon) = token.rfind(':') {
|
|
if colon > 0 {
|
|
groups.push(token[..colon].to_string());
|
|
}
|
|
}
|
|
}
|
|
if groups.is_empty() {
|
|
groups.push(group.to_string());
|
|
}
|
|
groups
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct Scanner {
|
|
files: FxHashMap<String, Option<Rc<FileMeta>>>,
|
|
key: String,
|
|
}
|
|
|
|
impl Scanner {
|
|
fn owner<'a>(&self, xref: &'a str, group: &'a str, priorities: &FxHashMap<String, u32>) -> &'a str {
|
|
let b = xref.as_bytes();
|
|
let Some(mut at) = xref.find(' ') else { return group };
|
|
let mut owner: Option<&str> = None;
|
|
let mut best = u32::MAX;
|
|
loop {
|
|
while b.get(at) == Some(&b' ') {
|
|
at += 1;
|
|
}
|
|
if at >= b.len() {
|
|
break;
|
|
}
|
|
let end = xref[at..].find(' ').map(|i| i + at);
|
|
// lastIndexOf(':', end ?? length): a colon at or before `end`.
|
|
let limit = end.unwrap_or(b.len());
|
|
let colon = xref[..limit.min(b.len())].rfind(':');
|
|
if let Some(colon) = colon.filter(|&c| c > at) {
|
|
let name = &xref[at..colon];
|
|
if let Some(&p) = priorities.get(name) {
|
|
if p < best {
|
|
best = p;
|
|
owner = Some(name);
|
|
}
|
|
}
|
|
}
|
|
match end {
|
|
Some(e) => at = e,
|
|
None => break,
|
|
}
|
|
}
|
|
owner.unwrap_or(group)
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn segment(&mut self, article: i64, subject: &str, from: &str, date: &str, message_id: &str, bytes: &str, xref: &str, group: &str) -> Option<Seg> {
|
|
if subject.is_empty() || message_id.is_empty() {
|
|
return None;
|
|
}
|
|
let sb = subject.as_bytes();
|
|
// After the last quote; without one, anywhere (unquoted_name finds the file name).
|
|
let name_end = subject.rfind('"');
|
|
let counter = find_counter(sb, name_end)?;
|
|
self.key.clear();
|
|
self.key.push_str(from);
|
|
self.key.push('\u{1}');
|
|
self.key.push_str(&subject[..counter.index]);
|
|
self.key.push('\u{0}');
|
|
self.key.push_str(&subject[counter.index + counter.length..]);
|
|
let file = match self.files.get(self.key.as_str()) {
|
|
Some(f) => f.clone(),
|
|
None => {
|
|
let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new);
|
|
self.files.insert(self.key.clone(), f.clone());
|
|
f
|
|
}
|
|
};
|
|
let file = file?;
|
|
if counter.segment < 1 || counter.segment > file.total_parts as i64 {
|
|
return None;
|
|
}
|
|
let id = if message_id.starts_with('<') { message_id.get(1..message_id.len().saturating_sub(1)).unwrap_or("") } else { message_id };
|
|
Some(Seg {
|
|
number: counter.segment as u32,
|
|
message_id: id.to_string(),
|
|
bytes: js::parse_int(bytes).unwrap_or(0),
|
|
article,
|
|
posted_at: js::parse_date(date),
|
|
file,
|
|
})
|
|
}
|
|
}
|
|
|
|
fn describe(subject: &str, name_end: Option<usize>, counter: &Counter, from: &str, xref: &str, group: &str) -> Option<FileMeta> {
|
|
let quoted = match name_end {
|
|
Some(end) if end > 0 => subject[..end].rfind('"').map(|start| subject[start + 1..end].to_string()),
|
|
_ => None,
|
|
};
|
|
let name = match quoted {
|
|
Some(name) => name_beside_quotes(&name, &subject[..counter.index]).unwrap_or(name),
|
|
None => unquoted_name(&subject[..counter.index])?,
|
|
};
|
|
let name = name.as_str();
|
|
if name.is_empty() || counter.total < 1 {
|
|
return None;
|
|
}
|
|
let joined = format!("{}{}", &subject[..counter.index], &subject[counter.index + counter.length..]);
|
|
let base = js::trim(&js::collapse_spaces(&joined)).to_string();
|
|
let file_key = sha1_hex(&format!("{from}\n{base}\n{}", counter.total));
|
|
let release = names::release_identity(from, &base, name);
|
|
Some(FileMeta {
|
|
file_key,
|
|
name: name.to_string(),
|
|
poster: from.to_string(),
|
|
subject: base,
|
|
total_parts: counter.total as u32,
|
|
groups: xref_groups(xref, group),
|
|
release,
|
|
})
|
|
}
|
|
|
|
/// EXTENSION: /[^.\s]\.[A-Za-z0-9]{2,4}$/
|
|
fn has_extension(s: &str) -> bool {
|
|
let Some(dot) = s.rfind('.') else { return false };
|
|
let ext = &s.as_bytes()[dot + 1..];
|
|
if !(2..=4).contains(&ext.len()) || !ext.iter().all(|c| c.is_ascii_alphanumeric()) {
|
|
return false;
|
|
}
|
|
s[..dot].chars().last().is_some_and(|c| c != '.' && !js::is_space(c))
|
|
}
|
|
|
|
/// The first / - \d{1,3} - / in `text`: the index of its " - ".
|
|
fn first_track(text: &str) -> Option<usize> {
|
|
let b = text.as_bytes();
|
|
let mut from = 0;
|
|
while let Some(i) = text[from..].find(" - ").map(|i| i + from) {
|
|
let digits = b[i + 3..].iter().take_while(|c| c.is_ascii_digit()).count();
|
|
if (1..=3).contains(&digits) && b[i + 3 + digits..].starts_with(b" - ") {
|
|
return Some(i);
|
|
}
|
|
from = i + 1;
|
|
}
|
|
None
|
|
}
|
|
|
|
/// nameBesideQuotes (lib/headers/subject.ts): a file name elsewhere when the quoted text has no extension.
|
|
pub fn name_beside_quotes(quoted: &str, before_counter: &str) -> Option<String> {
|
|
if has_extension(quoted) {
|
|
return None;
|
|
}
|
|
// A stray quote took the last pair: the last quoted part that is a file name.
|
|
let mut named = None;
|
|
let mut rest = before_counter;
|
|
while let Some(open) = rest.find('"') {
|
|
let Some(close) = rest[open + 1..].find('"') else { break };
|
|
let part = &rest[open + 1..open + 1 + close];
|
|
if has_extension(part) {
|
|
named = Some(part);
|
|
}
|
|
rest = &rest[open + 1 + close + 1..];
|
|
}
|
|
if let Some(part) = named {
|
|
return Some(part.to_string());
|
|
}
|
|
// .replace(/"[^"]*"/g, ' ')
|
|
let mut text = String::with_capacity(before_counter.len());
|
|
let mut rest = before_counter;
|
|
while let Some(open) = rest.find('"') {
|
|
match rest[open + 1..].find('"') {
|
|
Some(close) => {
|
|
text.push_str(&rest[..open]);
|
|
text.push(' ');
|
|
rest = &rest[open + 1 + close + 1..];
|
|
}
|
|
None => break,
|
|
}
|
|
}
|
|
text.push_str(rest);
|
|
unquoted_name(&text).filter(|n| has_extension(n))
|
|
}
|
|
|
|
/// unquotedName (lib/headers/subject.ts): the file name of a subject without a quoted one, from the text before its counter.
|
|
/// /\.(jpe?g|png|gif|bmp|webp|tiff?)$/i
|
|
fn is_image_name(name: &str) -> bool {
|
|
let lower = name.to_ascii_lowercase();
|
|
[".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tif", ".tiff"].iter().any(|e| lower.ends_with(e))
|
|
}
|
|
|
|
/// DATE_ONLY (lib/headers/subject.ts): a file name that is a date alone ("15_07_2015.rar").
|
|
fn date_only(name: &str) -> bool {
|
|
static DATE_ONLY: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
|
|
regex::Regex::new(r"^(?:[0-9]{1,2}[-_.][0-9]{1,2}[-_.](?:19|20)[0-9]{2}|(?:19|20)[0-9]{2}[-_.][0-9]{1,2}[-_.][0-9]{1,2})\.[A-Za-z0-9]{2,4}$").unwrap()
|
|
});
|
|
DATE_ONLY.is_match(name)
|
|
}
|
|
|
|
pub fn unquoted_name(before_counter: &str) -> Option<String> {
|
|
let mut text = before_counter.trim_end_matches(js::is_space);
|
|
if text.len() >= 4 && text.is_char_boundary(text.len() - 4) && text[text.len() - 4..].eq_ignore_ascii_case("yenc") {
|
|
text = text[..text.len() - 4].trim_end_matches(js::is_space);
|
|
}
|
|
if let Some(quote) = text.find('"') {
|
|
if text[quote + 1..].contains('"') {
|
|
return None;
|
|
}
|
|
let before = &text[..quote];
|
|
if let Some(bracket) = before.rfind('[') {
|
|
let inside = js::trim(&before[bracket + 1..]);
|
|
// replace(/^\d+\/\d+ +/, '')
|
|
let b = inside.as_bytes();
|
|
let mut i = 0;
|
|
while i < b.len() && b[i].is_ascii_digit() {
|
|
i += 1;
|
|
}
|
|
let mut stripped = inside;
|
|
if i > 0 && b.get(i) == Some(&b'/') {
|
|
let mut j = i + 1;
|
|
while j < b.len() && b[j].is_ascii_digit() {
|
|
j += 1;
|
|
}
|
|
if j > i + 1 && b.get(j) == Some(&b' ') {
|
|
while j < b.len() && b[j] == b' ' {
|
|
j += 1;
|
|
}
|
|
stripped = &inside[j..];
|
|
}
|
|
}
|
|
if has_extension(stripped) {
|
|
return Some(stripped.to_string());
|
|
}
|
|
}
|
|
let word = &before[before.rfind(' ').map_or(0, |i| i + 1)..];
|
|
return has_extension(word).then(|| word.to_string());
|
|
}
|
|
// A single-quoted name at the end: /'([^']*[^.\s']\.[A-Za-z0-9]{2,4})'$/
|
|
if let Some(inner) = text.strip_suffix('\'') {
|
|
if let Some(open) = inner.rfind('\'') {
|
|
let part = &inner[open + 1..];
|
|
if has_extension(part) {
|
|
return Some(part.to_string());
|
|
}
|
|
}
|
|
}
|
|
if has_extension(text) {
|
|
let bracket = text.rfind("] ").map_or(0, |i| i + 2);
|
|
// A track or part number (/ - \d{1,3} - /): the name starts there; else after the last " - ".
|
|
let dash = first_track(text).or_else(|| text.rfind(" - ")).map_or(0, |i| i + 3);
|
|
// Or after an arrow ("… -> Exclusive-tgirls_000000-lg.jpg").
|
|
let arrow = text.rfind("-> ").map_or(0, |i| i + 3);
|
|
let cut = bracket.max(dash).max(arrow);
|
|
// A date alone after the cut ("… [ettv] 15_07_2015.rar"): the whole text is the name.
|
|
let name = if date_only(js::trim(&text[cut..])) { js::trim(text) } else { js::trim(&text[cut..]) };
|
|
// An image after numbered text without a separator: its last word ("<abm> … AB: part2 0573-1002 043.jpg").
|
|
// Not an index alone ("Teen Tease Self Shooter sad1 (10).jpg").
|
|
if cut == 0 && is_image_name(name) && name.contains(' ') {
|
|
let space = name.rfind(' ').unwrap();
|
|
let last = &name[space + 1..];
|
|
if name[..space].trim_end_matches(js::is_space).ends_with(|c: char| c.is_ascii_digit()) && !last.starts_with('(') {
|
|
return Some(last.to_string());
|
|
}
|
|
}
|
|
// Ending in a host is no file name: the one in brackets or a word follows.
|
|
if !is_host(name) {
|
|
return (!name.is_empty() && has_extension(name)).then(|| name.to_string());
|
|
}
|
|
}
|
|
// A name in brackets: the last /\[([^[\]]*[^.\s[\]]\.[A-Za-z0-9]{2,4})\]/.
|
|
let mut bracketed = None;
|
|
for (open, _) in text.match_indices('[') {
|
|
let inside = &text[open + 1..];
|
|
let Some(close) = inside.find(']') else { break };
|
|
let part = &inside[..close];
|
|
if !part.contains('[') && has_extension(part) {
|
|
bracketed = Some(part);
|
|
}
|
|
}
|
|
if let Some(part) = bracketed {
|
|
return Some(part.to_string());
|
|
}
|
|
text.split(' ').find(|w| has_extension(w) && !is_host(w)).map(str::to_string)
|
|
}
|
|
|
|
/// /www\./i: a host ("www.libranews.nl"), whose ".nl" is no extension.
|
|
fn is_host(s: &str) -> bool {
|
|
s.to_ascii_lowercase().contains("www.")
|
|
}
|
|
|
|
/// One line of an XOVER reply: number, subject, from, date, message-id, bytes, xref (scanOverviewText).
|
|
fn scan_overview_text(text: &str, mut line: impl FnMut(Option<i64>, &str, &str, &str, &str, &str, &str)) {
|
|
let b = text.as_bytes();
|
|
let mut start = 0;
|
|
while start < b.len() {
|
|
let end = memchr::memchr(b'\n', &b[start..]).map(|i| i + start).unwrap_or(b.len());
|
|
let stop = if end > start && b[end - 1] == b'\r' { end - 1 } else { end };
|
|
if stop > start {
|
|
let l = &text[start..stop];
|
|
let lb = l.as_bytes();
|
|
let mut tabs = [usize::MAX; 9];
|
|
let mut n = 0;
|
|
for i in memchr::memchr_iter(b'\t', lb) {
|
|
tabs[n] = i;
|
|
n += 1;
|
|
if n == 9 {
|
|
break;
|
|
}
|
|
}
|
|
if n >= 5 {
|
|
let field = |a: usize, b: usize| -> &str {
|
|
if a == usize::MAX {
|
|
""
|
|
} else {
|
|
&l[a + 1..if b == usize::MAX { l.len() } else { b }]
|
|
}
|
|
};
|
|
let mut xref = if n >= 8 { field(tabs[7], tabs[8]) } else { "" };
|
|
if let Some(rest) = xref.strip_prefix("Xref:") {
|
|
xref = rest.strip_prefix(' ').unwrap_or(rest);
|
|
}
|
|
line(js::parse_int(&l[..tabs[0]]), &l[tabs[0] + 1..tabs[1]], field(tabs[1], tabs[2]), field(tabs[2], tabs[3]), field(tabs[3], tabs[4]), field(tabs[5], tabs[6]), xref);
|
|
}
|
|
}
|
|
start = end + 1;
|
|
}
|
|
}
|
|
|
|
/// A re-upload's copy of a file is kept apart (FileAggregator, uploadGapMs).
|
|
pub const UPLOAD_GAP_MS: i64 = 3 * 60 * 60 * 1000;
|
|
|
|
struct OpenFile {
|
|
parts: Vec<Seg>,
|
|
seen: FxHashSet<u32>,
|
|
total: u32,
|
|
touched: u64,
|
|
/// Posting times seen.
|
|
earliest: Option<i64>,
|
|
latest: Option<i64>,
|
|
}
|
|
|
|
impl OpenFile {
|
|
fn distance(&self, t: Option<i64>) -> i64 {
|
|
match (t, self.earliest, self.latest) {
|
|
(Some(t), Some(e), Some(l)) => 0.max(e - t).max(t - l),
|
|
_ => 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct Aggregator {
|
|
/// Per file key its open copies, oldest first.
|
|
open: FxHashMap<String, Vec<OpenFile>>,
|
|
batch: u64,
|
|
}
|
|
|
|
impl Aggregator {
|
|
fn add(&mut self, segments: Vec<Seg>) -> Vec<Vec<Seg>> {
|
|
self.batch += 1;
|
|
let batch = self.batch;
|
|
let mut touched: Vec<String> = Vec::new();
|
|
let mut touched_set: FxHashSet<String> = FxHashSet::default();
|
|
for s in segments {
|
|
let key = s.file.file_key.clone();
|
|
if !touched_set.contains(key.as_str()) {
|
|
touched_set.insert(key.clone());
|
|
touched.push(key.clone());
|
|
}
|
|
let copies = self.open.entry(key).or_default();
|
|
let t = s.posted_at;
|
|
// A copy lacking the part, the nearest in time.
|
|
let mut chosen: Option<usize> = None;
|
|
let mut best = i64::MAX;
|
|
for (i, copy) in copies.iter().enumerate() {
|
|
if copy.seen.contains(&s.number) {
|
|
continue;
|
|
}
|
|
let d = copy.distance(t);
|
|
if d < best {
|
|
best = d;
|
|
chosen = Some(i);
|
|
}
|
|
}
|
|
let index = match chosen {
|
|
Some(i) => i,
|
|
None => {
|
|
if let Some(near) = copies.iter_mut().find(|c| c.distance(t) <= UPLOAD_GAP_MS) {
|
|
near.touched = batch;
|
|
continue;
|
|
}
|
|
copies.push(OpenFile { parts: Vec::new(), seen: FxHashSet::default(), total: s.file.total_parts, touched: batch, earliest: None, latest: None });
|
|
copies.len() - 1
|
|
}
|
|
};
|
|
let file = &mut copies[index];
|
|
file.seen.insert(s.number);
|
|
if let Some(t) = t {
|
|
if file.earliest.is_none_or(|e| t < e) {
|
|
file.earliest = Some(t);
|
|
}
|
|
if file.latest.is_none_or(|l| t > l) {
|
|
file.latest = Some(t);
|
|
}
|
|
}
|
|
file.touched = batch;
|
|
file.parts.push(s);
|
|
}
|
|
let mut complete = Vec::new();
|
|
for key in touched {
|
|
let copies = self.open.get_mut(&key).unwrap();
|
|
let mut i = 0;
|
|
while i < copies.len() {
|
|
if (copies[i].parts.len() as u32) < copies[i].total {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
let mut f = copies.remove(i);
|
|
f.parts.sort_by_key(|s| s.number);
|
|
complete.push(f.parts);
|
|
}
|
|
if copies.is_empty() {
|
|
self.open.remove(&key);
|
|
}
|
|
}
|
|
complete
|
|
}
|
|
|
|
/// Files idle for `idle` batches (all when 0): their segments, per file key (its copies one after another).
|
|
fn evict(&mut self, idle: u64) -> Vec<Vec<Seg>> {
|
|
let batch = self.batch;
|
|
let mut out = Vec::new();
|
|
let keys: Vec<String> = self.open.keys().cloned().collect();
|
|
for key in keys {
|
|
let copies = self.open.get_mut(&key).unwrap();
|
|
let mut gone: Vec<Seg> = Vec::new();
|
|
let mut i = 0;
|
|
while i < copies.len() {
|
|
if idle > 0 && batch - copies[i].touched < idle {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
gone.extend(copies.remove(i).parts);
|
|
}
|
|
if copies.is_empty() {
|
|
self.open.remove(&key);
|
|
}
|
|
if !gone.is_empty() {
|
|
out.push(gone);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
}
|
|
|
|
/// uploadsOf: a key's files split where posting pauses for more than UPLOAD_GAP_MS after the latest article so far.
|
|
fn uploads_of(files: Vec<Vec<Seg>>) -> Vec<Vec<Vec<Seg>>> {
|
|
if files.len() < 2 {
|
|
return vec![files];
|
|
}
|
|
let first: Vec<Option<i64>> = files.iter().map(|f| posted_of(f)).collect();
|
|
let last: Vec<Option<i64>> = files.iter().map(|f| f.iter().filter_map(|s| s.posted_at).max()).collect();
|
|
let mut dated: Vec<usize> = (0..files.len()).filter(|&i| first[i].is_some()).collect();
|
|
if dated.len() < 2 {
|
|
return vec![files];
|
|
}
|
|
dated.sort_by_key(|&i| (first[i].unwrap(), i));
|
|
let mut upload = vec![0usize; files.len()];
|
|
let mut current = 0;
|
|
let mut end = last[dated[0]].unwrap();
|
|
for &i in &dated[1..] {
|
|
if first[i].unwrap() - end > UPLOAD_GAP_MS {
|
|
current += 1;
|
|
end = last[i].unwrap();
|
|
} else if last[i].unwrap() > end {
|
|
end = last[i].unwrap();
|
|
}
|
|
upload[i] = current;
|
|
}
|
|
if current == 0 {
|
|
return vec![files];
|
|
}
|
|
let mut out: Vec<Vec<Vec<Seg>>> = (0..=current).map(|_| Vec::new()).collect();
|
|
for (i, f) in files.into_iter().enumerate() {
|
|
out[upload[i]].push(f);
|
|
}
|
|
out
|
|
}
|
|
|
|
pub struct RangeIndexer {
|
|
group: String,
|
|
rules: Rules,
|
|
start: i64,
|
|
end: i64,
|
|
priorities: FxHashMap<String, u32>,
|
|
idle: u64,
|
|
chunk_articles: i64,
|
|
scanner: Scanner,
|
|
aggregator: Aggregator,
|
|
batch: Batch,
|
|
counts: FxHashMap<(&'static str, String), (u64, u64)>,
|
|
count_order: Vec<(&'static str, String)>,
|
|
completed: FxHashMap<String, Vec<Vec<Seg>>>,
|
|
completed_order: Vec<String>,
|
|
}
|
|
|
|
impl RangeIndexer {
|
|
/// `resume_at`: where this worker starts (a range taken over midway); its first batch starts there.
|
|
pub fn new(group: &str, start: i64, end: i64, resume_at: i64, rules: Rules, idle: u64) -> Self {
|
|
RangeIndexer {
|
|
group: group.to_string(),
|
|
start,
|
|
end,
|
|
priorities: rules.priorities.clone(),
|
|
rules,
|
|
idle,
|
|
chunk_articles: 0,
|
|
scanner: Scanner::default(),
|
|
aggregator: Aggregator::default(),
|
|
batch: Batch { from: resume_at, next: resume_at, ..Default::default() },
|
|
counts: FxHashMap::default(),
|
|
count_order: Vec::new(),
|
|
completed: FxHashMap::default(),
|
|
completed_order: Vec::new(),
|
|
}
|
|
}
|
|
|
|
pub fn add_text(&mut self, text: &str, next: i64) {
|
|
let mut segments: Vec<Seg> = Vec::new();
|
|
let mut crossposts: Vec<String> = Vec::new();
|
|
let mut obfuscated = 0u64;
|
|
let mut headers = 0u64;
|
|
{
|
|
let group = self.group.clone();
|
|
let scanner = &mut self.scanner;
|
|
let priorities = &self.priorities;
|
|
scan_overview_text(text, |n, subject, from, date, message_id, bytes, xref| {
|
|
let Some(n) = n else { return };
|
|
if message_id.is_empty() {
|
|
return;
|
|
}
|
|
headers += 1;
|
|
let owner = if xref.is_empty() { group.as_str() } else { scanner.owner(xref, &group, priorities) };
|
|
if owner != group {
|
|
crossposts.push(owner.to_string());
|
|
return;
|
|
}
|
|
match scanner.segment(n, subject, from, date, message_id, bytes, xref, &group) {
|
|
Some(s) => segments.push(s),
|
|
None => obfuscated += 1,
|
|
}
|
|
});
|
|
}
|
|
self.batch.headers += headers;
|
|
for owner in crossposts {
|
|
self.count("crosspost", owner, 1, 0);
|
|
}
|
|
if obfuscated > 0 {
|
|
self.count("obfuscated", String::new(), obfuscated, 0);
|
|
}
|
|
self.end_chunk(segments, next);
|
|
}
|
|
|
|
fn end_chunk(&mut self, segments: Vec<Seg>, next: i64) {
|
|
let t = std::time::Instant::now();
|
|
let done = self.aggregator.add(segments);
|
|
unsafe { PROF[3] += t.elapsed().as_nanos() as u64; }
|
|
self.files(done);
|
|
if self.chunk_articles == 0 {
|
|
self.chunk_articles = next - self.batch.from;
|
|
}
|
|
let t = std::time::Instant::now();
|
|
let gone = self.aggregator.evict(self.idle);
|
|
self.cut_off(gone, false);
|
|
unsafe { PROF[4] += t.elapsed().as_nanos() as u64; }
|
|
self.batch.next = next;
|
|
}
|
|
|
|
pub fn finish(&mut self) {
|
|
let gone = self.aggregator.evict(0);
|
|
self.cut_off(gone, true);
|
|
self.batch.next = self.end + 1;
|
|
}
|
|
|
|
/// Files completed and fragments cut since the last take (RangeIndexer.pending).
|
|
pub fn pending(&self) -> usize {
|
|
self.batch.fragments.len() + self.completed.values().map(|l| l.len()).sum::<usize>()
|
|
}
|
|
|
|
pub fn take(&mut self) -> Batch {
|
|
let t = std::time::Instant::now();
|
|
self.judge_releases();
|
|
unsafe { PROF[5] += t.elapsed().as_nanos() as u64; }
|
|
self.scanner.files.clear();
|
|
let next = self.batch.next;
|
|
let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() });
|
|
for id in self.count_order.drain(..) {
|
|
let (articles, files) = self.counts.remove(&id).unwrap();
|
|
out.counts.push((id.0, id.1, articles, files));
|
|
}
|
|
out
|
|
}
|
|
|
|
fn files(&mut self, files: Vec<Vec<Seg>>) {
|
|
for file in files {
|
|
let key = release_key_of(&file[0]);
|
|
match self.completed.get_mut(&key) {
|
|
Some(list) => list.push(file),
|
|
None => {
|
|
self.completed_order.push(key.clone());
|
|
self.completed.insert(key, vec![file]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn judge_releases(&mut self) {
|
|
let order = std::mem::take(&mut self.completed_order);
|
|
let mut by_release = std::mem::take(&mut self.completed);
|
|
for key in order {
|
|
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
|
let judged: Vec<JudgedFile> = list
|
|
.iter()
|
|
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id })
|
|
.collect();
|
|
match names::judge(&key, &judged, &self.rules) {
|
|
Verdict::Keep => {
|
|
let candidates = candidates_of(&key, &list, upload);
|
|
self.batch.releases.push(chunk_of(&key, &list, upload));
|
|
self.batch.candidates.extend(candidates);
|
|
}
|
|
Verdict::Drop { rule, key: rule_key, sample } => {
|
|
let segments: usize = list.iter().map(|f| f.len()).sum();
|
|
self.count(rule, rule_key.clone(), segments as u64, list.len() as u64);
|
|
if sample {
|
|
self.batch.samples.push((chunk_of(&key, &list, upload), rule, rule_key));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn cut_off(&mut self, files: Vec<Vec<Seg>>, at_end: bool) {
|
|
let window = self.idle as i64 * self.chunk_articles.max(1);
|
|
for list in files {
|
|
if list.len() == 1 && is_random_file(&list[0].file.name) {
|
|
let tool = names::upload_tool(&list[0].message_id);
|
|
self.count("per-article", tool.to_string(), 1, 0);
|
|
continue;
|
|
}
|
|
let min_part = list.iter().map(|s| s.number).min().unwrap();
|
|
let max_part = list.iter().map(|s| s.number).max().unwrap();
|
|
let first_article = list.iter().map(|s| s.article).min().unwrap();
|
|
let cut_at_start = min_part > 1 && first_article < self.start + window;
|
|
let cut_at_end = at_end && max_part < list[0].file.total_parts;
|
|
if !cut_at_start && !cut_at_end {
|
|
self.count("incomplete", String::new(), list.len() as u64, 1);
|
|
continue;
|
|
}
|
|
let f = &list[0].file;
|
|
let mut fragment = Fragment {
|
|
file_key: f.file_key.clone(),
|
|
name: f.name.clone(),
|
|
poster: f.poster.clone(),
|
|
subject: f.subject.clone(),
|
|
total_parts: f.total_parts,
|
|
groups: f.groups.clone(),
|
|
release: f.release.clone(),
|
|
posted_at: None,
|
|
segments: Vec::with_capacity(list.len()),
|
|
};
|
|
for s in &list {
|
|
fragment.segments.push((s.number, s.message_id.clone(), s.bytes, s.article));
|
|
if let Some(t) = s.posted_at {
|
|
if fragment.posted_at.is_none_or(|p| t < p) {
|
|
fragment.posted_at = Some(t);
|
|
}
|
|
}
|
|
}
|
|
self.batch.fragments.push(fragment);
|
|
}
|
|
}
|
|
|
|
fn count(&mut self, rule: &'static str, key: String, articles: u64, files: u64) {
|
|
match self.counts.get_mut(&(rule, key.clone())) {
|
|
Some(c) => {
|
|
c.0 += articles;
|
|
c.1 += files;
|
|
}
|
|
None => {
|
|
self.count_order.push((rule, key.clone()));
|
|
self.counts.insert((rule, key), (articles, files));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn release_key_of(s: &Seg) -> String {
|
|
// A named file always has a release (releaseIdentity isn't null unless weak).
|
|
s.file.release.key.clone()
|
|
}
|
|
|
|
fn posted_of(segments: &[Seg]) -> Option<i64> {
|
|
segments.iter().filter_map(|s| s.posted_at).min()
|
|
}
|
|
|
|
fn chunk_of(key: &str, files: &[Vec<Seg>], upload: usize) -> ReleaseChunk {
|
|
let first = &files[0][0].file;
|
|
let chunk_files: Vec<ChunkFile> = files
|
|
.iter()
|
|
.map(|f| {
|
|
let m = &f[0].file;
|
|
ChunkFile { name: &m.name, poster: &m.poster, subject: &m.subject, posted_at: posted_of(f), groups: &m.groups, segments: f.iter().map(|s| (s.number, s.message_id.as_str(), s.bytes)).collect() }
|
|
})
|
|
.collect();
|
|
let mut segments = 0;
|
|
let mut bytes = 0;
|
|
let mut first_article = i64::MAX;
|
|
let mut last_article = 0;
|
|
let mut posted_at: Option<i64> = None;
|
|
let mut posted_last: Option<i64> = None;
|
|
for f in files {
|
|
for s in f {
|
|
if let Some(t) = s.posted_at {
|
|
if posted_last.is_none_or(|p| t > p) {
|
|
posted_last = Some(t);
|
|
}
|
|
}
|
|
segments += 1;
|
|
bytes += s.bytes;
|
|
first_article = first_article.min(s.article);
|
|
last_article = last_article.max(s.article);
|
|
}
|
|
if let Some(t) = posted_of(f) {
|
|
if posted_at.is_none_or(|p| t < p) {
|
|
posted_at = Some(t);
|
|
}
|
|
}
|
|
}
|
|
ReleaseChunk {
|
|
key: key.to_string(),
|
|
upload,
|
|
name: first.release.name.clone(),
|
|
source: first.release.source,
|
|
file_total: files.iter().find_map(|f| f[0].file.release.file_total_truthy()),
|
|
poster: first.poster.clone(),
|
|
groups: first.groups.clone(),
|
|
posted_at,
|
|
posted_last,
|
|
files: files.len(),
|
|
segments,
|
|
bytes,
|
|
first_article,
|
|
last_article,
|
|
readable: chunk_files.iter().any(|f| !is_random_file(f.name)),
|
|
has_par2: chunk_files.iter().any(|f| names::is_par2(f.name)),
|
|
payload: encode_chunk(&chunk_files),
|
|
subjects: chunk_files.iter().take(SUBJECTS).map(|f| f.subject.to_string()).collect(),
|
|
}
|
|
}
|
|
|
|
fn candidates_of(key: &str, files: &[Vec<Seg>], upload: usize) -> Vec<Candidate> {
|
|
let mut out = Vec::new();
|
|
let mut tail: Option<(i64, &Vec<Seg>)> = None;
|
|
let c = |kind, s: &Seg| Candidate { release_key: key.to_string(), upload, kind, message_id: s.message_id.clone(), file: s.file.name.clone() };
|
|
for f in files {
|
|
let name = &f[0].file.name;
|
|
if let Some(volume) = names::seven_zip_volume(name).filter(|&v| v > 1) {
|
|
if tail.is_none_or(|(v, _)| volume > v) {
|
|
tail = Some((volume, f));
|
|
}
|
|
continue;
|
|
}
|
|
if names::is_archive_start(name) {
|
|
out.push(c("archive", &f[0]));
|
|
} else if names::is_par2(name) && !names::is_later_volume(name) {
|
|
out.push(c("par2", &f[0]));
|
|
}
|
|
}
|
|
if let Some((_, f)) = tail {
|
|
out.push(Candidate { release_key: key.to_string(), upload, kind: "7z-tail", message_id: f.last().unwrap().message_id.clone(), file: f[0].file.name.clone() });
|
|
}
|
|
if out.is_empty() && files.len() == 1 && is_random_file(&files[0][0].file.name) {
|
|
out.push(c("lone", &files[0][0]));
|
|
}
|
|
out.truncate(4);
|
|
out
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
const GROUP: &str = "alt.binaries.test";
|
|
|
|
fn line(n: i64, subject: &str, minutes: i64) -> String {
|
|
let date = chrono::DateTime::from_timestamp_millis(1_791_194_400_000 + minutes * 60_000).unwrap().format("%a, %d %b %Y %H:%M:%S +0000");
|
|
format!("{n}\t{subject}\tp <p@x>\t{date}\t<u{n}@test>\t\t1000\t10\tXref: news.example {GROUP}:{n}")
|
|
}
|
|
|
|
#[test]
|
|
fn uploads_split_as_in_the_typescript() {
|
|
let mut priorities = FxHashMap::default();
|
|
priorities.insert(GROUP.to_string(), 1);
|
|
let mut ix = RangeIndexer::new(GROUP, 1, 10_000, 1, Rules::defaults(priorities), 5);
|
|
let mut lines: Vec<String> = (0..5).map(|i| line(100 + i, &format!(r#"Big.Release [1/2] - "Big.Release.mkv" yEnc ({}/5)"#, i + 1), i * 60)).collect();
|
|
lines.push(line(200, r#"Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)"#, 245));
|
|
lines.extend((0..5).map(|i| line(300 + i, &format!(r#"Big.Release [1/2] - "Big.Release.mkv" yEnc ({}/5)"#, i + 1), 1440 + i)));
|
|
ix.add_text(&lines.join("\r\n"), 10_001);
|
|
ix.finish();
|
|
let batch = ix.take();
|
|
let got: Vec<(usize, usize)> = batch.releases.iter().map(|r| (r.upload, r.files)).collect();
|
|
assert_eq!(got, vec![(0, 2), (1, 1)]);
|
|
assert_eq!(batch.releases[0].posted_last.unwrap() - batch.releases[0].posted_at.unwrap(), 245 * 60_000);
|
|
assert_eq!(batch.releases[0].key, batch.releases[1].key);
|
|
}
|
|
|
|
#[test]
|
|
fn copies_and_fills_as_in_the_typescript() {
|
|
let hour = 3_600_000i64;
|
|
let meta = Rc::new(FileMeta {
|
|
file_key: "a".into(),
|
|
name: "a.bin".into(),
|
|
poster: "p".into(),
|
|
subject: "s".into(),
|
|
total_parts: 3,
|
|
groups: vec![],
|
|
release: names::release_identity("p", "s", "a.bin"),
|
|
});
|
|
let at = |number: u32, h: f64, id: &str| Seg { file: meta.clone(), number, message_id: id.into(), bytes: 100, article: 0, posted_at: Some((h * hour as f64) as i64) };
|
|
let mut agg = Aggregator::default();
|
|
assert!(agg.add(vec![at(1, 10.0, "a1"), at(2, 10.0, "a2"), at(1, 10.1, "resend")]).is_empty());
|
|
assert!(agg.add(vec![at(1, 34.0, "b1"), at(2, 34.0, "b2")]).is_empty());
|
|
assert_eq!(agg.open.get("a").unwrap().len(), 2);
|
|
let second = agg.add(vec![at(3, 34.2, "b3")]);
|
|
assert_eq!(second[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["b1", "b2", "b3"]);
|
|
let first = agg.add(vec![at(3, 58.0, "a3")]);
|
|
assert_eq!(first[0].iter().map(|s| s.message_id.as_str()).collect::<Vec<_>>(), vec!["a1", "a2", "a3"]);
|
|
assert!(agg.open.is_empty());
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod unquoted_tests {
|
|
use super::unquoted_name;
|
|
|
|
#[test]
|
|
fn file_names_of_subjects_without_a_quoted_one() {
|
|
let cases = [
|
|
("DVDFab 8.2.0.5.rar ", Some("DVDFab 8.2.0.5.rar")),
|
|
("q3B9OrpKiCxUnzgtUGnp.part023.rar ", Some("q3B9OrpKiCxUnzgtUGnp.part023.rar")),
|
|
("New_Super_Mario_Bros._[Keygen].exe ", Some("New_Super_Mario_Bros._[Keygen].exe")),
|
|
("Emily_18_-_Bed.part01.rar Emily 18 bed video FULL NUDE ", Some("Emily_18_-_Bed.part01.rar")),
|
|
("[33/117] 674a37548b09.dat ", Some("674a37548b09.dat")),
|
|
("A Strange Light from the East - Eastern Influenced Western Music (1965-72) - CD2 - 07 - Rascals - Sattva.mp3 ", Some("07 - Rascals - Sattva.mp3")),
|
|
("pal! [Mivina high(1).wmv\" yEnc] ", Some("Mivina high(1).wmv")),
|
|
("[03/21 Geroellheimer - S03E16.part01.rar\" yEnc ", Some("Geroellheimer - S03E16.part01.rar")),
|
|
("<Group> FP -> {EXL01~P~004} -> Alexia (3) -> VF -> [0001 of 0076] -> Exclusive-tgirls_000000-lg.jpg ", Some("Exclusive-tgirls_000000-lg.jpg")),
|
|
("Ballers.2014.S01E01.HDTV.x264-KILLERS[ettv] 15_07_2015.rar ", Some("Ballers.2014.S01E01.HDTV.x264-KILLERS[ettv] 15_07_2015.rar")),
|
|
("Computer Bild - Ausgabe 152015 - 15_07_2015.rar ", Some("Computer Bild - Ausgabe 152015 - 15_07_2015.rar")),
|
|
("<abm> <Duke> here's a little thanksgiving present: AB: part2 0573-1002 043.jpg ", Some("043.jpg")),
|
|
("xlady post You, Me And Dupree www.x2l.nl [001/106] 'You, Me And Dupree.nfo'", Some("You, Me And Dupree.nfo")),
|
|
("[01/77][LIBRATEAM_M2219009.par2] Onbeperkt downloaden met een snelheid van 1,22 MB p.s. en dat voor maar 7,95 p.m. www.libranews.nl", Some("LIBRATEAM_M2219009.par2")),
|
|
("patensas post -^-[Mnozil Brass - Seven.nzb]-^-www.usenet2day.cc -^-[000/107]-^-www.libranews.nl-^- yEnc", Some("Mnozil Brass - Seven.nzb")),
|
|
("[7w] - 9BbEV2MrTi59GBTEFlVrssBrXNwXf2SK9cQEeDjg yEnc ", None),
|
|
("Re: Will somebody answer the question? ", None),
|
|
];
|
|
for (text, want) in cases {
|
|
assert_eq!(unquoted_name(text).as_deref(), want, "{text}");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod beside_quotes_tests {
|
|
use super::name_beside_quotes;
|
|
|
|
#[test]
|
|
fn a_name_beside_a_quoted_word() {
|
|
assert_eq!(name_beside_quotes("Official", "My \"Official\" remaining set's of Inna, Please help up to #69 if you can 025.jpg [25/74] - yEnc ").as_deref(), Some("025.jpg"));
|
|
assert_eq!(name_beside_quotes("Dr. Lust", "\"Dr. Lust\" - Lily Marlene, Vanessa Del Rio, Scorpio? - yEnc - [12/21] - Ed_DRL_VDR_LM.part09.rar ").as_deref(), Some("Ed_DRL_VDR_LM.part09.rar"));
|
|
assert_eq!(name_beside_quotes("1f0554e5fec14b8b05ca", "77cd6d03d19281a775f5bdc5 [2/8] - \"1f0554e5fec14b8b05ca\" yEnc ").as_deref(), None);
|
|
assert_eq!(name_beside_quotes("x.rar", "\"x.rar\" other.zip ").as_deref(), None);
|
|
}
|
|
}
|
|
}
|