diff --git a/tools/edge-bench-rs/Cargo.lock b/tools/edge-bench-rs/Cargo.lock index 1239213..6419d0e 100644 --- a/tools/edge-bench-rs/Cargo.lock +++ b/tools/edge-bench-rs/Cargo.lock @@ -121,6 +121,7 @@ dependencies = [ "chrono", "flate2", "md-5", + "mimalloc", "regex", "rustc-hash", "sha1", @@ -238,6 +239,15 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "log" version = "0.4.34" @@ -260,6 +270,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "num-traits" version = "0.2.19" diff --git a/tools/edge-bench-rs/Cargo.toml b/tools/edge-bench-rs/Cargo.toml index ce3c056..719ec94 100644 --- a/tools/edge-bench-rs/Cargo.toml +++ b/tools/edge-bench-rs/Cargo.toml @@ -9,6 +9,7 @@ base64 = "0.22" chrono = "0.4" flate2 = { version = "1", default-features = false, features = ["zlib-rs"] } md-5 = "0.10" +mimalloc = "0.1.52" regex = "1" rustc-hash = "2" sha1 = "0.10" diff --git a/tools/edge-bench-rs/src/indexer.rs b/tools/edge-bench-rs/src/indexer.rs index c2765dd..f203f6b 100644 --- a/tools/edge-bench-rs/src/indexer.rs +++ b/tools/edge-bench-rs/src/indexer.rs @@ -8,6 +8,8 @@ use crate::names::{self, is_random_file, JudgedFile, Release, Verdict}; use rustc_hash::{FxHashMap, FxHashSet}; use std::rc::Rc; +pub static mut PROF: [u64; 8] = [0; 8]; + pub struct FileMeta { pub file_key: String, pub name: String, @@ -193,7 +195,9 @@ impl Scanner { let file = match self.files.get(self.key.as_str()) { Some(f) => f.clone(), None => { + let t = std::time::Instant::now(); let f = describe(subject, name_end, &counter, from, xref, group).map(Rc::new); + unsafe { PROF[0] += t.elapsed().as_nanos() as u64; PROF[1] += 1; } self.files.insert(self.key.clone(), f.clone()); f } @@ -208,7 +212,7 @@ impl Scanner { message_id: id.to_string(), bytes: js::parse_int(bytes).unwrap_or(0), article, - posted_at: js::parse_date(date), + posted_at: { let t = std::time::Instant::now(); let d = js::parse_date(date); unsafe { PROF[2] += t.elapsed().as_nanos() as u64; } d }, file, }) } @@ -405,13 +409,17 @@ impl RangeIndexer { } fn end_chunk(&mut self, segments: Vec, next: i64) { + let t = std::time::Instant::now(); let done = self.aggregator.add(segments); + unsafe { PROF[3] += t.elapsed().as_nanos() as u64; } self.files(done); if self.chunk_articles == 0 { self.chunk_articles = next - self.batch.from; } + let t = std::time::Instant::now(); let gone = self.aggregator.evict(self.idle); self.cut_off(gone, false); + unsafe { PROF[4] += t.elapsed().as_nanos() as u64; } self.batch.next = next; } @@ -422,7 +430,9 @@ impl RangeIndexer { } pub fn take(&mut self) -> Batch { + let t = std::time::Instant::now(); self.judge_releases(); + unsafe { PROF[5] += t.elapsed().as_nanos() as u64; } self.scanner.files.clear(); let next = self.batch.next; let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() }); diff --git a/tools/edge-bench-rs/src/main.rs b/tools/edge-bench-rs/src/main.rs index a795135..5b5f25e 100644 --- a/tools/edge-bench-rs/src/main.rs +++ b/tools/edge-bench-rs/src/main.rs @@ -18,6 +18,10 @@ use rustc_hash::FxHashMap; use std::io::{Read, Write}; use std::time::Instant; +// Many small strings (message-ids, names): the system allocator was a measurable share. +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + fn inflate(raw: &[u8]) -> Vec { let mut out = Vec::with_capacity(raw.len() * 6); if raw.starts_with(&[0x1f, 0x8b]) { @@ -200,6 +204,9 @@ fn main() { } } let total = inflate_s + index_s; + #[allow(static_mut_refs)] + let p = unsafe { indexer::PROF }; + eprintln!("describe {:.2} s ({} calls), dates {:.2} s, aggregate {:.2} s, evict+cutoff {:.2} s, judge+encode {:.2} s", p[0] as f64 / 1e9, p[1], p[2] as f64 / 1e9, p[3] as f64 / 1e9, p[4] as f64 / 1e9, p[5] as f64 / 1e9); println!("{group} {start}–{end}: {} chunks, {} MB compressed, {} MB text", files.len(), compressed / 1_000_000, text_bytes / 1_000_000); println!( "{} headers in {:.2} s ({:.0}/s on one core; wall {:.2} s)", diff --git a/tools/edge-bench-rs/src/names.rs b/tools/edge-bench-rs/src/names.rs index 936f581..afbd3c9 100644 --- a/tools/edge-bench-rs/src/names.rs +++ b/tools/edge-bench-rs/src/names.rs @@ -44,8 +44,9 @@ static SUFFIXES: LazyLock> = LazyLock::new(|| { ] }); -/// fileStem: per-file suffixes stripped, repeatedly. -pub fn file_stem(name: &str) -> String { +/// fileStem with the regexes, as in the TypeScript (the reference for file_stem's tests). +#[allow(dead_code)] +pub fn file_stem_re(name: &str) -> String { let mut stem = js::trim(name).to_string(); let mut changed = true; while changed { @@ -70,7 +71,9 @@ static YENC_END: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+) static YENC: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b")); static SEPARATORS: LazyLock = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+")); -pub fn subject_remainder(subject: &str) -> String { +/// subjectRemainder with the regexes, as in the TypeScript (the reference for subject_remainder's tests). +#[allow(dead_code)] +pub fn subject_remainder_re(subject: &str) -> String { let s = replace("ED, subject, " ", true); let s = replace(&COUNTERS, &s, " ", true); let s = replace(&YENC_END, &s, " ", false); @@ -79,6 +82,223 @@ pub fn subject_remainder(subject: &str) -> String { js::trim(&s).to_string() } +fn eq_ci(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_ascii_lowercase() == *y) +} + +/// Ends in `suffix` (lowercase), ASCII case-insensitively. +fn ends_ci(s: &[u8], suffix: &[u8]) -> bool { + s.len() >= suffix.len() && eq_ci(&s[s.len() - suffix.len()..], suffix) +} + +/// ASCII digits ending at `end`. +fn digits_back(s: &[u8], end: usize) -> usize { + s[..end].iter().rev().take_while(|c| c.is_ascii_digit()).count() +} + +/// `.` + one of `exts` at the end: the length without it. +fn strip_ext(s: &[u8], exts: &[&[u8]]) -> Option { + for ext in exts { + let n = ext.len() + 1; + if s.len() >= n && s[s.len() - n] == b'.' && ends_ci(s, ext) { + return Some(s.len() - n); + } + } + None +} + +/// `\.\d{3}$`: the length without it. +fn strip_three_digits(s: &[u8]) -> Option { + let n = s.len(); + (n >= 4 && s[n - 4] == b'.' && s[n - 3..].iter().all(u8::is_ascii_digit)).then(|| n - 4) +} + +const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"]; +const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"]; +const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"]; +const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"]; + +/// SUFFIXES[rule] once: the stem's new length when it matches. +fn strip_suffix_rule(s: &[u8], rule: usize) -> Option { + let n = s.len(); + match rule { + // \.vol\d+[+-]\d+\.par2$ + 0 => { + if !ends_ci(s, b".par2") { + return None; + } + let d2 = digits_back(s, n - 5); + let sign = (n - 5).checked_sub(d2 + 1).filter(|_| d2 > 0)?; + if !matches!(s[sign], b'+' | b'-') { + return None; + } + let d1 = digits_back(s, sign); + let v = sign.checked_sub(d1 + 4).filter(|_| d1 > 0)?; + eq_ci(&s[v..v + 4], b".vol").then_some(v) + } + 1 => ends_ci(s, b".par2").then(|| n - 5), + // \.part\d+\.rar$ + 2 => { + if !ends_ci(s, b".rar") { + return None; + } + let d = digits_back(s, n - 4); + let p = (n - 4).checked_sub(d + 5).filter(|_| d > 0)?; + eq_ci(&s[p..p + 5], b".part").then_some(p) + } + // \.r\d{2,3}$ + 3 => { + let d = digits_back(s, n); + let p = n.checked_sub(d + 2).filter(|_| (2..=3).contains(&d))?; + eq_ci(&s[p..p + 2], b".r").then_some(p) + } + 4 => ends_ci(s, b".rar").then(|| n - 4), + // \.7z(\.\d{3})?$ and \.zip(\.\d{3})?$ + 5 | 6 => { + let ext: &[u8] = if rule == 5 { b".7z" } else { b".zip" }; + if let Some(m) = strip_three_digits(s) { + if ends_ci(&s[..m], ext) { + return Some(m - ext.len()); + } + } + ends_ci(s, ext).then(|| n - ext.len()) + } + // (?<=\.(7z|…))\.\d{3}$: only the volume number goes. + 7 => { + let m = strip_three_digits(s)?; + strip_ext(&s[..m], &SPLIT_EXTS).map(|_| m) + } + 8 => strip_ext(s, &SIDE_EXTS), + 9 => strip_ext(s, &[b"tar"]), + 10 => strip_ext(s, &PACK_EXTS), + 11 => strip_ext(s, &MEDIA_EXTS), + _ => None, + } +} + +/// fileStem: per-file suffixes stripped, repeatedly. The SUFFIXES regexes by hand: with them, a +/// file's description was most of the indexing time. +pub fn file_stem(name: &str) -> String { + let stem = js::trim(name); + let b = stem.as_bytes(); + let mut len = b.len(); + let mut changed = true; + while changed { + changed = false; + for rule in 0..12 { + if let Some(next) = strip_suffix_rule(&b[..len], rule) { + if next > 0 && next < len { + len = next; + changed = true; + } + } + } + } + // The suffixes are ASCII: `len` is a character boundary. + stem[..len].to_string() +} + +fn is_word(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' +} + +/// "yEnc" (any case) at `i` with a word boundary (ASCII, as in JS) on both sides. +fn yenc_at(b: &[u8], i: usize) -> bool { + i + 4 <= b.len() && eq_ci(&b[i..i + 4], b"yenc") && (i == 0 || !is_word(b[i - 1])) && (i + 4 == b.len() || !is_word(b[i + 4])) +} + +/// `(\s+\d+)?\s*$` +fn yenc_tail(rest: &str) -> bool { + let after_spaces = rest.trim_start_matches(js::is_space); + if after_spaces.chars().all(js::is_space) { + return true; + } + if after_spaces.len() == rest.len() { + return false; + } + let after_digits = after_spaces.trim_start_matches(|c: char| c.is_ascii_digit()); + after_digits.len() < after_spaces.len() && after_digits.chars().all(js::is_space) +} + +/// subjectRemainder by hand: the same five replacements and trim. +pub fn subject_remainder(subject: &str) -> String { + let b = subject.as_bytes(); + // "[^"]*" → ' ' + let mut s1: Vec = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'"' { + if let Some(j) = b[i + 1..].iter().position(|&c| c == b'"') { + s1.push(b' '); + i += j + 2; + continue; + } + } + s1.push(b[i]); + i += 1; + } + // \[\d+/\d+\] and \(\d+/\d+\) → ' ' + let mut s2: Vec = Vec::with_capacity(s1.len()); + let mut i = 0; + while i < s1.len() { + let close = match s1[i] { + b'[' => Some(b']'), + b'(' => Some(b')'), + _ => None, + }; + if let Some(close) = close { + let a = s1[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); + let slash = i + 1 + a; + if a > 0 && s1.get(slash) == Some(&b'/') { + let d = s1[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); + if d > 0 && s1.get(slash + 1 + d) == Some(&close) { + s2.push(b' '); + i = slash + 2 + d; + continue; + } + } + } + s2.push(s1[i]); + i += 1; + } + // Only ASCII was replaced, between whole characters. + let mut s3 = String::from_utf8(s2).unwrap(); + // \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits) + let b3 = s3.as_bytes(); + if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) { + s3.truncate(at); + s3.push(' '); + } + // \byEnc\b → ' ', then runs of \s, -, –, ., :, | → ' ' + let b3 = s3.as_bytes(); + let mut s4 = String::with_capacity(s3.len()); + let mut separator = false; + let mut i = 0; + while i < b3.len() { + if yenc_at(b3, i) { + // Its " " starts or joins a separator run. + if !separator { + s4.push(' '); + separator = true; + } + i += 4; + continue; + } + let c = s3[i..].chars().next().unwrap(); + if js::is_space(c) || matches!(c, '-' | '.' | ':' | '|' | '\u{2013}') { + if !separator { + s4.push(' '); + separator = true; + } + } else { + s4.push(c); + separator = false; + } + i += c.len_utf8(); + } + js::trim(&s4).to_string() +} + fn is_random_token(token: &str) -> bool { let t = token.strip_prefix('[').unwrap_or(token); let t = t.strip_suffix(']').unwrap_or(t); @@ -341,6 +561,34 @@ mod tests { assert!(!names_release("elys de81bcabef5faf6a")); } + #[test] + fn by_hand_as_the_regexes() { + let names = [ + "x.mkv.vol03+04.par2", "x.part07.rar", "Movie.H.264.mkv.001", "a.7z.002", "a.7z", "b.zip.001", "B.ZIP", "c.r00", "c.r1234", + "d.vol1+2.par2", "d.vol+2.par2", ".par2", "e.tar.gz", "f.mkv", "g.m3u8", "h.bin.001", " spaced.rar ", "i.PART01.RAR", "j.part.rar", + "k.nfo.txt", "noext", "ünï.cödé.mkv.001", "l.ts.123", "m.r12.par2", "n.vol12-34.PAR2", "o.7z.1234", "p.123", "q.r1", ".rar.rar", + ]; + for n in names { + assert_eq!(file_stem(n), file_stem_re(n), "{n}"); + } + let subjects = [ + r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#, + r#"[1/3] - "a.par2" yEnc (1/1) 123 "#, + r#"yEnc "x" yenc yEncx (1/2)"#, + r#"Show – S01E01 | 1080p : x "a" "b"#, + r#"name (12/3x) [4/] "q" YENC 12 34"#, + " lead\u{a0}nbsp yEnc\u{3000}99 ", + r#"elys-de81bcabef5faf6a - [Nyh5Q07Y] - "x.mkv" yEnc"#, + "a_yEnc b yEnc_c yEnc", + "yEnc yEnc 5", + "x yEnc 12 yEnc", + "", + ]; + for s in subjects { + assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}"); + } + } + #[test] fn random_names() { assert!(looks_random("prvjr7hapd0i1ny14pz4"));