Rebuild: edge-bench-rs, the edge worker's indexing in Rust, for comparison
The RangeIndexer path of edge:bench (inflate, scan, owner rule, segments, file aggregation, cut-off, rules, release identity, chunk encoding v2, probe candidates) as a Rust binary over a captured range, with JS semantics where they matter (\s, trim, \b, parseInt, lastIndexOf, UTF-16 lengths). check.ts runs the TypeScript RangeIndexer on the same capture and compares every output: identical on all three captures (releases and their decoded payloads, samples, candidates, rule counts, fragments), after mapping Date headers ending in UTC to GMT (chrono follows RFC 5322; Date.parse takes UTC, a quarter of a.b.boneless). Compressed payloads differ by <0.1% in size (zstd framing), not in content. First run on boneless 2026: 360k headers/s and 228 MB peak (TypeScript: ~217k/s, ~0.7-0.8 GB). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
42b6cac3ee
commit
bacc9aa3b3
9 files changed
+2192
No files matched your search
@@ -0,0 +1,358 @@
|
||||
//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools.
|
||||
//! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u`
|
||||
//! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS
|
||||
//! `\s` (which includes Unicode spaces) is spelled out as SP.
|
||||
|
||||
use crate::js::{self, len16, sha1_hex};
|
||||
use md5::{Digest, Md5};
|
||||
use regex::bytes::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// JS `\s` as a byte pattern.
|
||||
const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)";
|
||||
|
||||
fn re(pattern: &str) -> Regex {
|
||||
Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap()
|
||||
}
|
||||
|
||||
/// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all).
|
||||
fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String {
|
||||
let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) };
|
||||
// The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8.
|
||||
String::from_utf8(out.into_owned()).unwrap()
|
||||
}
|
||||
|
||||
fn test(re: &Regex, s: &str) -> bool {
|
||||
re.is_match(s.as_bytes())
|
||||
}
|
||||
|
||||
static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
vec![
|
||||
(re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""),
|
||||
(re(r"(?i)\.par2$"), ""),
|
||||
(re(r"(?i)\.part[0-9]+\.rar$"), ""),
|
||||
(re(r"(?i)\.r[0-9]{2,3}$"), ""),
|
||||
(re(r"(?i)\.rar$"), ""),
|
||||
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||||
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||||
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.[0-9]{3}$"), "$1"),
|
||||
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||||
(re(r"(?i)\.tar$"), ""),
|
||||
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||||
]
|
||||
});
|
||||
|
||||
/// fileStem: per-file suffixes stripped, repeatedly.
|
||||
pub fn file_stem(name: &str) -> String {
|
||||
let mut stem = js::trim(name).to_string();
|
||||
let mut changed = true;
|
||||
while changed {
|
||||
changed = false;
|
||||
for (suffix, with) in SUFFIXES.iter() {
|
||||
if !suffix.is_match(stem.as_bytes()) {
|
||||
continue;
|
||||
}
|
||||
let next = replace(suffix, &stem, with, false);
|
||||
if next != stem && !next.is_empty() {
|
||||
stem = next;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
stem
|
||||
}
|
||||
|
||||
static QUOTED: LazyLock<Regex> = LazyLock::new(|| re(r#""[^"]*""#));
|
||||
static COUNTERS: LazyLock<Regex> = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)"));
|
||||
static YENC_END: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$"));
|
||||
static YENC: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b"));
|
||||
static SEPARATORS: LazyLock<Regex> = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+"));
|
||||
|
||||
pub fn subject_remainder(subject: &str) -> String {
|
||||
let s = replace("ED, subject, " ", true);
|
||||
let s = replace(&COUNTERS, &s, " ", true);
|
||||
let s = replace(&YENC_END, &s, " ", false);
|
||||
let s = replace(&YENC, &s, " ", true);
|
||||
let s = replace(&SEPARATORS, &s, " ", true);
|
||||
js::trim(&s).to_string()
|
||||
}
|
||||
|
||||
fn is_random_token(token: &str) -> bool {
|
||||
let t = token.strip_prefix('[').unwrap_or(token);
|
||||
let t = t.strip_suffix(']').unwrap_or(t);
|
||||
if len16(t) < 6 {
|
||||
return false;
|
||||
}
|
||||
let digit = t.bytes().any(|b| b.is_ascii_digit());
|
||||
if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
return true;
|
||||
}
|
||||
digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-'))
|
||||
}
|
||||
|
||||
pub fn is_post_token(remainder: &str) -> bool {
|
||||
let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect();
|
||||
let n = len16(&compact);
|
||||
(6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric())
|
||||
}
|
||||
|
||||
static NAME_LIKE: LazyLock<Regex> = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}"));
|
||||
|
||||
pub fn names_release(remainder: &str) -> bool {
|
||||
let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect();
|
||||
test(&NAME_LIKE, &words.join(" "))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Release {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub source: &'static str,
|
||||
/// The [i/M] total as parsed (0 is falsy in the TypeScript).
|
||||
pub file_total: Option<i64>,
|
||||
}
|
||||
|
||||
impl Release {
|
||||
pub fn file_total_truthy(&self) -> Option<i64> {
|
||||
self.file_total.filter(|&n| n != 0)
|
||||
}
|
||||
}
|
||||
|
||||
static FILE_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]"));
|
||||
|
||||
fn hash(parts: &[&str]) -> String {
|
||||
sha1_hex(&parts.join("\n"))
|
||||
}
|
||||
|
||||
/// releaseIdentity for a named (not weak) file: never null.
|
||||
pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||
let file_total = FILE_COUNTER
|
||||
.captures(subject.as_bytes())
|
||||
.and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap()));
|
||||
let stem = file_stem(name);
|
||||
let remainder = subject_remainder(subject);
|
||||
if names_release(&remainder) {
|
||||
return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total };
|
||||
}
|
||||
if let Some(total) = file_total.filter(|&n| n != 0) {
|
||||
if is_post_token(&remainder) {
|
||||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||||
}
|
||||
}
|
||||
Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total }
|
||||
}
|
||||
|
||||
static GENERIC: LazyLock<Regex> = LazyLock::new(|| {
|
||||
re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$")
|
||||
});
|
||||
|
||||
fn is_generic_name(name: &str) -> bool {
|
||||
test(&GENERIC, js::trim(name))
|
||||
}
|
||||
|
||||
/// looksRandom.
|
||||
pub fn looks_random(name: &str) -> bool {
|
||||
// name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
let name = match name.rfind('.') {
|
||||
Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i],
|
||||
_ => name,
|
||||
};
|
||||
if is_generic_name(name) {
|
||||
return true;
|
||||
}
|
||||
let b = name.as_bytes();
|
||||
let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit());
|
||||
let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit());
|
||||
if b.len() >= 16 && hex(b) {
|
||||
return true;
|
||||
}
|
||||
if let Some(dot) = name.find('.') {
|
||||
let (h, d) = (&b[..dot], &b[dot + 1..]);
|
||||
if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) {
|
||||
return true;
|
||||
}
|
||||
{
|
||||
let core = name.strip_suffix(" n").unwrap_or(name).as_bytes();
|
||||
if core.len() >= 6 && digits(core) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count();
|
||||
let n_digit = count(u8::is_ascii_digit);
|
||||
let n_lower = count(u8::is_ascii_lowercase);
|
||||
let n_upper = count(u8::is_ascii_uppercase);
|
||||
let alnum = b.iter().all(|c| c.is_ascii_alphanumeric());
|
||||
if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 {
|
||||
return true;
|
||||
}
|
||||
if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 {
|
||||
return true;
|
||||
}
|
||||
if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 {
|
||||
return false;
|
||||
}
|
||||
n_digit > 0 || n_upper >= 6
|
||||
}
|
||||
|
||||
pub fn is_random_file(name: &str) -> bool {
|
||||
looks_random(&file_stem(name))
|
||||
}
|
||||
|
||||
static SEVEN_SPLIT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$"));
|
||||
static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||||
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||||
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||||
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||||
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||||
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||||
|
||||
pub fn is_par2(name: &str) -> bool {
|
||||
test(&PAR2, name)
|
||||
}
|
||||
|
||||
pub fn seven_zip_volume(name: &str) -> Option<i64> {
|
||||
if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) {
|
||||
return js::parse_int(std::str::from_utf8(&c[1]).unwrap());
|
||||
}
|
||||
if test(&SEVEN, name) {
|
||||
Some(0)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_archive_start(name: &str) -> bool {
|
||||
if test(&BIN, name) {
|
||||
return true;
|
||||
}
|
||||
if matches!(seven_zip_volume(name), Some(0) | Some(1)) {
|
||||
return true;
|
||||
}
|
||||
test(&PART1, name) || (test(&RAR, name) && !test(&PART, name))
|
||||
}
|
||||
|
||||
pub fn is_later_volume(name: &str) -> bool {
|
||||
if let Some(c) = PART.captures(name.as_bytes()) {
|
||||
return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1;
|
||||
}
|
||||
test(&OLD_RAR, name) || test(&PAR2_VOL, name)
|
||||
}
|
||||
|
||||
static TOOLS: LazyLock<Vec<(&'static str, Regex)>> = LazyLock::new(|| {
|
||||
vec![
|
||||
("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")),
|
||||
("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")),
|
||||
("ngPost", re(r"^[0-9a-f]{32}@ngPost$")),
|
||||
("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")),
|
||||
("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")),
|
||||
("JBinUp", re(r"@JBin(?:Up|Down)\.local$")),
|
||||
("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")),
|
||||
("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")),
|
||||
("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")),
|
||||
]
|
||||
});
|
||||
|
||||
pub fn upload_tool(message_id: &str) -> &'static str {
|
||||
if message_id.is_empty() {
|
||||
return "other";
|
||||
}
|
||||
for (tool, pattern) in TOOLS.iter() {
|
||||
if test(pattern, message_id) {
|
||||
return tool;
|
||||
}
|
||||
}
|
||||
"other"
|
||||
}
|
||||
|
||||
/// explored: a release in the sampled share of a rule's drops.
|
||||
pub fn explored(key: &str, percent: f64) -> bool {
|
||||
if percent <= 0.0 {
|
||||
return false;
|
||||
}
|
||||
let d = Md5::digest(key.as_bytes());
|
||||
let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]);
|
||||
((n % 10_000) as f64) < percent * 100.0
|
||||
}
|
||||
|
||||
static SIDECAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$"));
|
||||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||||
|
||||
pub struct JudgedFile<'a> {
|
||||
pub name: &'a str,
|
||||
pub total_parts: u32,
|
||||
pub groups: &'a [String],
|
||||
pub first_message_id: &'a str,
|
||||
}
|
||||
|
||||
pub enum Verdict {
|
||||
Keep,
|
||||
Drop { rule: &'static str, key: String, sample: bool },
|
||||
}
|
||||
|
||||
/// RuleSet.judge with DEFAULT_RULES (no dropped signatures).
|
||||
pub fn judge(release_key: &str, files: &[JudgedFile]) -> Verdict {
|
||||
const DROPPED_TOOLS: [&str; 5] = ["ngPost", "hex32", "yuun", "JBinUp", "random"];
|
||||
let sample = explored(release_key, 1.0);
|
||||
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||||
if !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||||
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||||
let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
|
||||
return Verdict::Drop { rule: "executable", key, sample };
|
||||
}
|
||||
if files.iter().any(|f| !is_random_file(f.name)) {
|
||||
return Verdict::Keep;
|
||||
}
|
||||
let first = &files[0];
|
||||
let tool = upload_tool(first.first_message_id);
|
||||
let group = first.groups.first().map(String::as_str).unwrap_or("");
|
||||
if DROPPED_TOOLS.contains(&tool) {
|
||||
return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample };
|
||||
}
|
||||
if files.iter().all(|f| f.total_parts <= 3) {
|
||||
return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample };
|
||||
}
|
||||
Verdict::Keep
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn stems() {
|
||||
assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x");
|
||||
assert_eq!(file_stem("x.part07.rar"), "x");
|
||||
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||||
assert_eq!(file_stem("a.7z.002"), "a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remainders() {
|
||||
assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release");
|
||||
assert!(names_release("Some Release"));
|
||||
assert!(!names_release("elys de81bcabef5faf6a"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_names() {
|
||||
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||||
assert!(looks_random("RmISakytFwXSCEzjVEpaKk"));
|
||||
assert!(!looks_random("Some.Show.S01E01.1080p"));
|
||||
assert!(looks_random("VIDEO_TS"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tools() {
|
||||
assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu");
|
||||
assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost");
|
||||
assert_eq!(upload_tool("x@y"), "other");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user