//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools. //! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u` //! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS //! `\s` (which includes Unicode spaces) is spelled out as SP. use crate::js::{self, len16, sha1_hex}; use md5::{Digest, Md5}; use regex::bytes::Regex; use std::sync::LazyLock; /// JS `\s` as a byte pattern. const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)"; fn re(pattern: &str) -> Regex { Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap() } /// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all). fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String { let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) }; // The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8. String::from_utf8(out.into_owned()).unwrap() } fn test(re: &Regex, s: &str) -> bool { re.is_match(s.as_bytes()) } static SUFFIXES: LazyLock> = LazyLock::new(|| { vec![ (re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""), (re(r"(?i)\.par2$"), ""), (re(r"(?i)\.part[0-9]+\.rar$"), ""), (re(r"(?i)\.r[0-9]{2,3}$"), ""), (re(r"(?i)\.rar$"), ""), (re(r"(?i)\.7z(\.[0-9]{3})?$"), ""), (re(r"(?i)\.zip(\.[0-9]{3})?$"), ""), // JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back. (re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.[0-9]{3}$"), "$1"), (re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""), (re(r"(?i)\.tar$"), ""), (re(r"(?i)\.(zst|gz|xz|bz2)$"), ""), (re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""), ] }); /// fileStem with the regexes, as in the TypeScript (the reference for file_stem's tests). #[allow(dead_code)] pub fn file_stem_re(name: &str) -> String { let mut stem = js::trim(name).to_string(); let mut changed = true; while changed { changed = false; for (suffix, with) in SUFFIXES.iter() { if !suffix.is_match(stem.as_bytes()) { continue; } let next = replace(suffix, &stem, with, false); if next != stem && !next.is_empty() { stem = next; changed = true; } } } stem } static QUOTED: LazyLock = LazyLock::new(|| re(r#""[^"]*""#)); static COUNTERS: LazyLock = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)")); static YENC_END: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$")); static YENC: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b")); static SEPARATORS: LazyLock = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+")); /// subjectRemainder with the regexes, as in the TypeScript (the reference for subject_remainder's tests). #[allow(dead_code)] pub fn subject_remainder_re(subject: &str) -> String { let s = replace("ED, subject, " ", true); let s = replace(&COUNTERS, &s, " ", true); let s = remove_counters_and_sizes(&s); let s = replace(&YENC_END, &s, " ", false); let s = replace(&YENC, &s, " ", true); let s = replace(&SEPARATORS, &s, " ", true); finish_remainder(&s) } fn eq_ci(a: &[u8], b: &[u8]) -> bool { a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_ascii_lowercase() == *y) } /// Ends in `suffix` (lowercase), ASCII case-insensitively. fn ends_ci(s: &[u8], suffix: &[u8]) -> bool { s.len() >= suffix.len() && eq_ci(&s[s.len() - suffix.len()..], suffix) } /// ASCII digits ending at `end`. fn digits_back(s: &[u8], end: usize) -> usize { s[..end].iter().rev().take_while(|c| c.is_ascii_digit()).count() } /// `.` + one of `exts` at the end: the length without it. fn strip_ext(s: &[u8], exts: &[&[u8]]) -> Option { for ext in exts { let n = ext.len() + 1; if s.len() >= n && s[s.len() - n] == b'.' && ends_ci(s, ext) { return Some(s.len() - n); } } None } /// `\.\d{3}$`: the length without it. fn strip_three_digits(s: &[u8]) -> Option { let n = s.len(); (n >= 4 && s[n - 4] == b'.' && s[n - 3..].iter().all(u8::is_ascii_digit)).then(|| n - 4) } const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"]; const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"]; const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"]; const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"]; /// SUFFIXES[rule] once: the stem's new length when it matches. fn strip_suffix_rule(s: &[u8], rule: usize) -> Option { let n = s.len(); match rule { // \.vol\d+[+-]\d+\.par2$ 0 => { if !ends_ci(s, b".par2") { return None; } let d2 = digits_back(s, n - 5); let sign = (n - 5).checked_sub(d2 + 1).filter(|_| d2 > 0)?; if !matches!(s[sign], b'+' | b'-') { return None; } let d1 = digits_back(s, sign); let v = sign.checked_sub(d1 + 4).filter(|_| d1 > 0)?; eq_ci(&s[v..v + 4], b".vol").then_some(v) } 1 => ends_ci(s, b".par2").then(|| n - 5), // \.part\d+\.rar$ 2 => { if !ends_ci(s, b".rar") { return None; } let d = digits_back(s, n - 4); let p = (n - 4).checked_sub(d + 5).filter(|_| d > 0)?; eq_ci(&s[p..p + 5], b".part").then_some(p) } // \.r\d{2,3}$ 3 => { let d = digits_back(s, n); let p = n.checked_sub(d + 2).filter(|_| (2..=3).contains(&d))?; eq_ci(&s[p..p + 2], b".r").then_some(p) } 4 => ends_ci(s, b".rar").then(|| n - 4), // \.7z(\.\d{3})?$ and \.zip(\.\d{3})?$ 5 | 6 => { let ext: &[u8] = if rule == 5 { b".7z" } else { b".zip" }; if let Some(m) = strip_three_digits(s) { if ends_ci(&s[..m], ext) { return Some(m - ext.len()); } } ends_ci(s, ext).then(|| n - ext.len()) } // (?<=\.(7z|…))\.\d{3}$: only the volume number goes. 7 => { let m = strip_three_digits(s)?; strip_ext(&s[..m], &SPLIT_EXTS).map(|_| m) } 8 => strip_ext(s, &SIDE_EXTS), 9 => strip_ext(s, &[b"tar"]), 10 => strip_ext(s, &PACK_EXTS), 11 => strip_ext(s, &MEDIA_EXTS), _ => None, } } /// fileStem: per-file suffixes stripped, repeatedly. The SUFFIXES regexes by hand: with them, a /// file's description was most of the indexing time. pub fn file_stem(name: &str) -> String { let stem = js::trim(name); let b = stem.as_bytes(); let mut len = b.len(); let mut changed = true; while changed { changed = false; for rule in 0..12 { if let Some(next) = strip_suffix_rule(&b[..len], rule) { if next > 0 && next < len { len = next; changed = true; } } } } // The suffixes are ASCII: `len` is a character boundary. stem[..len].to_string() } fn is_word(c: u8) -> bool { c.is_ascii_alphanumeric() || c == b'_' } /// "yEnc" (any case) at `i` with a word boundary (ASCII, as in JS) on both sides. fn yenc_at(b: &[u8], i: usize) -> bool { i + 4 <= b.len() && eq_ci(&b[i..i + 4], b"yenc") && (i == 0 || !is_word(b[i - 1])) && (i + 4 == b.len() || !is_word(b[i + 4])) } /// `(\s+\d+)?\s*$` fn yenc_tail(rest: &str) -> bool { let after_spaces = rest.trim_start_matches(js::is_space); if after_spaces.chars().all(js::is_space) { return true; } if after_spaces.len() == rest.len() { return false; } let after_digits = after_spaces.trim_start_matches(|c: char| c.is_ascii_digit()); after_digits.len() < after_spaces.len() && after_digits.chars().all(js::is_space) } /// subjectRemainder by hand: the same five replacements and trim. pub fn subject_remainder(subject: &str) -> String { let b = subject.as_bytes(); // "[^"]*" → ' ' let mut s1: Vec = Vec::with_capacity(b.len()); let mut i = 0; while i < b.len() { if b[i] == b'"' { if let Some(j) = b[i + 1..].iter().position(|&c| c == b'"') { s1.push(b' '); i += j + 2; continue; } } s1.push(b[i]); i += 1; } // \[\d+/\d+\] and \(\d+/\d+\) → ' ' let mut s2: Vec = Vec::with_capacity(s1.len()); let mut i = 0; while i < s1.len() { let close = match s1[i] { b'[' => Some(b']'), b'(' => Some(b')'), _ => None, }; if let Some(close) = close { let a = s1[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); let slash = i + 1 + a; if a > 0 && s1.get(slash) == Some(&b'/') { let d = s1[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); if d > 0 && s1.get(slash + 1 + d) == Some(&close) { s2.push(b' '); i = slash + 2 + d; continue; } } } s2.push(s1[i]); i += 1; } // Only ASCII was replaced, between whole characters. let mut s3 = remove_counters_and_sizes(&String::from_utf8(s2).unwrap()); // \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits) let b3 = s3.as_bytes(); if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) { s3.truncate(at); s3.push(' '); } // \byEnc\b → ' ', then runs of \s, -, –, ., :, | → ' ' let b3 = s3.as_bytes(); let mut s4 = String::with_capacity(s3.len()); let mut separator = false; let mut i = 0; while i < b3.len() { if yenc_at(b3, i) { // Its " " starts or joins a separator run. if !separator { s4.push(' '); separator = true; } i += 4; continue; } let c = s3[i..].chars().next().unwrap(); if js::is_space(c) || matches!(c, '-' | '.' | ':' | '|' | '\u{2013}') { if !separator { s4.push(' '); separator = true; } } else { s4.push(c); separator = false; } i += c.len_utf8(); } finish_remainder(&s4) } static OF_COUNTER: LazyLock = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b[\])]?")); static DASH_COUNTER: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)-([0-9]+)\]")); static SLASH_COUNTER: LazyLock = LazyLock::new(|| re(r"\b([0-9]+)/([0-9]+)\b")); static ZERO_COUNTER: LazyLock = LazyLock::new(|| re(r"\b(0[0-9]*)-([0-9]+)\b")); static SIZES: LazyLock = LazyLock::new(|| re(r"(?i)\b[0-9]+(?:[.,][0-9]+)?SP*(?:[kmgt]i?b|bytes?)\b")); static EMPTY_BRACKETS: LazyLock = LazyLock::new(|| re(r" ?(?:\[ ?\]|\( ?\)) ?")); fn number(digits: &[u8]) -> f64 { std::str::from_utf8(digits).unwrap().parse::().unwrap() } /// `s.replace(/…/g, counter)`: the match goes when its first number is at most its second. fn replace_counters(re: &Regex, s: &str) -> String { if !re.is_match(s.as_bytes()) { return s.to_string(); } let out = re.replace_all(s.as_bytes(), |c: ®ex::bytes::Captures| if number(&c[1]) <= number(&c[2]) { b" ".to_vec() } else { c[0].to_vec() }); String::from_utf8(out.into_owned()).unwrap() } /// The written-out counters, file sizes and empty brackets subjectRemainder removes after the [i/M] ones. fn remove_counters_and_sizes(s: &str) -> String { let s = replace_counters(&OF_COUNTER, s); let s = replace_counters(&DASH_COUNTER, &s); let s = replace_counters(&SLASH_COUNTER, &s); let s = replace_counters(&ZERO_COUNTER, &s); replace(&SIZES, &s, " ", true) } /// The end of subjectRemainder: what's left of brackets, double spaces, trim (after the separators became single spaces). fn finish_remainder(s: &str) -> String { let s = if s.contains(['[', '(']) { replace(&EMPTY_BRACKETS, s, " ", true) } else { s.to_string() }; let mut out = String::with_capacity(s.len()); for c in s.chars() { if c == ' ' && out.ends_with(' ') { continue; } out.push(c); } js::trim(&out).to_string() } /// counterTotal: the subject's file counter total, first form found first. pub fn counter_total(subject: &str) -> Option { let s = replace("ED, subject, " ", true); if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) { let total = c.get(2).or(c.get(4)).unwrap().as_bytes(); return Some(js_number_string(number(total))); } for re in [&*OF_COUNTER_OPEN, &*DASH_COUNTER, &*SLASH_COUNTER, &*ZERO_COUNTER] { for c in re.captures_iter(s.as_bytes()) { if number(&c[1]) <= number(&c[2]) { return Some(js_number_string(number(&c[2]))); } } } None } static COUNTERS_CAPTURE: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]|\(([0-9]+)/([0-9]+)\)")); /// The "of" counter without its closing bracket (as counterTotal's pattern). static OF_COUNTER_OPEN: LazyLock = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b")); /// `String(n)` of a counter's number (integers as JS prints them). fn js_number_string(n: f64) -> String { if n.fract() == 0.0 && n.abs() < 1e21 { format!("{}", n as i128) } else { format!("{n}") } } static IMAGE: LazyLock = LazyLock::new(|| re(r"(?i)\.(jpe?g|png|gif|bmp|webp|tiff?)$")); static AUDIO: LazyLock = LazyLock::new(|| re(r"(?i)\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])")); static DOCUMENT: LazyLock = LazyLock::new(|| re(r"(?i)\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$")); /// An image set's name without a counter: up to the last number, without numbers. fn image_prefix(stem: &str) -> String { let b = stem.as_bytes(); let cut = match b.iter().rposition(u8::is_ascii_digit) { Some(last) => b[..=last].iter().rposition(|c| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0), None => b.len(), }; let head = &stem[..cut]; let mut out = String::with_capacity(head.len()); let mut sep = false; for c in head.chars() { if c.is_ascii_digit() || js::is_space(c) || matches!(c, '.' | '_' | '-') { sep = true; } else { if sep && !out.is_empty() { out.push(' '); } out.push(c); sep = false; } } out } fn is_random_token(token: &str) -> bool { let t = token.strip_prefix('[').unwrap_or(token); let t = t.strip_suffix(']').unwrap_or(t); if len16(t) < 6 { return false; } let digit = t.bytes().any(|b| b.is_ascii_digit()); if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) { return true; } digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-')) } pub fn is_post_token(remainder: &str) -> bool { let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect(); let n = len16(&compact); (6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric()) } #[allow(dead_code)] static NAME_LIKE: LazyLock = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}")); pub fn names_release(remainder: &str) -> bool { let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect(); name_like(words.join(" ").as_bytes()) } #[derive(Debug, Clone)] pub struct Release { pub key: String, pub name: String, pub source: &'static str, /// The [i/M] total as parsed (0 is falsy in the TypeScript). pub file_total: Option, } impl Release { pub fn file_total_truthy(&self) -> Option { self.file_total.filter(|&n| n != 0) } } #[allow(dead_code)] static FILE_COUNTER: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]")); fn hash(parts: &[&str]) -> String { sha1_hex(&parts.join("\n")) } /// A name that says what it is: 12+ letters and digits, two words of 3+ letters (`describes`). fn describes(stem: &str) -> bool { let b = stem.as_bytes(); if b.iter().filter(|c| c.is_ascii_alphanumeric()).count() < 12 { return false; } // [A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,} let mut i = 0; while i < b.len() { let letters = b[i..].iter().take_while(|c| c.is_ascii_alphabetic()).count(); if letters == 0 { i += 1; continue; } let end = i + letters; if letters >= 3 { let gap = b[end..].iter().take_while(|c| !c.is_ascii_alphanumeric()).count(); if gap > 0 && b[end + gap..].iter().take_while(|c| c.is_ascii_alphabetic()).count() >= 3 { return true; } } i = end; } false } const TAG_WORDS: [&str; 21] = [ "www", "http", "https", "com", "net", "org", "info", "the", "and", "presents", "posted", "post", "powered", "yenc", "part", "rar", "par2", "nzb", "voor", "for", "from", ]; /// isPosterTag: the text outside the quotes shares no word with the file's stem. pub fn is_poster_tag(remainder: &str, stem: &str) -> bool { let lower = stem.to_lowercase(); let words: Vec<&str> = lower.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())).filter(|w| !w.is_empty()).collect(); let compact: String = words.concat(); let whole: std::collections::HashSet<&str> = words.iter().copied().collect(); // A per-post id, also inside a word ("FTD772544"). let b = remainder.as_bytes(); if b.windows(5).any(|w| w.iter().all(u8::is_ascii_digit)) { return false; } let rem = remainder.to_lowercase(); for word in rem.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())) { if !word.is_empty() && word.bytes().all(|c| c.is_ascii_digit()) { if word.len() >= 2 && whole.contains(word) { return false; } continue; } if word.len() < 3 || TAG_WORDS.contains(&word) { continue; } if if word.len() >= 4 { compact.contains(word) } else { whole.contains(word) } { return false; } } true } /// releaseIdentity for a named (not weak) file: never null. pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { // The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal). let post_total = file_counter_total(subject); let file_total = counter_total(subject).and_then(|t| t.parse::().ok()); let stem = file_stem(name); // A name that isn't quoted (unquoted_name) is taken out as the quotes would have been. let remainder = if subject.contains(&format!("\"{name}\"")) { subject_remainder(subject) } else { subject_remainder(&subject.replacen(name, " ", 1).replace('"', " ")) }; // A document or comic is a release of its own. if test(&DOCUMENT, name) { let shown = replace(&DOCUMENT, &stem, "", false); let name = if shown.is_empty() { stem.clone() } else { shown }; return Release { key: hash(&["stem", poster, &stem]), name, source: "stem", file_total }; } let image = test(&IMAGE, name); // The poster's constant tag outside the quotes: the file's stem says which upload it is (not for images and audio). let tag = !image && !test(&AUDIO, name) && describes(&stem) && !looks_random(&stem) && is_poster_tag(&remainder, &stem); let prefix = if image { image_prefix(&stem) } else { String::new() }; if !tag && names_release(&remainder) { let bound = counter_total(subject).unwrap_or_else(|| prefix.clone()); return Release { key: hash(&["subject", poster, &remainder, &bound]), name: remainder, source: "subject", file_total }; } if let Some(total) = post_total.filter(|&n| n != 0 && !tag) { if is_post_token(&remainder) { return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total }; } } if image { let name = if prefix.is_empty() { stem.clone() } else { prefix.clone() }; return Release { key: hash(&["stem", poster, &name]), name, source: "stem", file_total }; } Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total } } static GENERIC: LazyLock = LazyLock::new(|| { re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$") }); fn is_generic_name(name: &str) -> bool { test(&GENERIC, js::trim(name)) } /// looksRandom. pub fn looks_random(name: &str) -> bool { // name.replace(/\.[A-Za-z0-9]{1,4}$/, '') let name = match name.rfind('.') { Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i], _ => name, }; if is_generic_name(name) { return true; } let b = name.as_bytes(); let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit()); let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit()); if b.len() >= 16 && hex(b) { return true; } if let Some(dot) = name.find('.') { let (h, d) = (&b[..dot], &b[dot + 1..]); if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) { return true; } } if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) { return true; } { let core = name.strip_suffix(" n").unwrap_or(name).as_bytes(); if core.len() >= 6 && digits(core) { return true; } } let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count(); let n_digit = count(u8::is_ascii_digit); let n_lower = count(u8::is_ascii_lowercase); let n_upper = count(u8::is_ascii_uppercase); let alnum = b.iter().all(|c| c.is_ascii_alphanumeric()); if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 { return true; } if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 { return true; } // 51294P1CUKR2R35X: capitals and digits only, switching between them all over the token // (COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice). if b.len() >= 12 && b.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) && b.windows(2).filter(|w| w[0].is_ascii_digit() != w[1].is_ascii_digit()).count() >= 6 { return true; } if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 { return false; } n_digit > 0 || n_upper >= 6 } pub fn is_random_file(name: &str) -> bool { looks_random(&file_stem(name)) } static SEVEN_SPLIT: LazyLock = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$")); static SEVEN: LazyLock = LazyLock::new(|| re(r"(?i)\.7z$")); static PART1: LazyLock = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$")); static RAR: LazyLock = LazyLock::new(|| re(r"(?i)\.rar$")); static PART: LazyLock = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$")); static BIN: LazyLock = LazyLock::new(|| re(r"(?i)\.bin$")); static OLD_RAR: LazyLock = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$")); static PAR2_VOL: LazyLock = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$")); pub static PAR2: LazyLock = LazyLock::new(|| re(r"(?i)\.par2$")); pub fn is_par2(name: &str) -> bool { test(&PAR2, name) } pub fn seven_zip_volume(name: &str) -> Option { if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) { return js::parse_int(std::str::from_utf8(&c[1]).unwrap()); } if test(&SEVEN, name) { Some(0) } else { None } } pub fn is_archive_start(name: &str) -> bool { if test(&BIN, name) { return true; } if matches!(seven_zip_volume(name), Some(0) | Some(1)) { return true; } test(&PART1, name) || (test(&RAR, name) && !test(&PART, name)) } pub fn is_later_volume(name: &str) -> bool { if let Some(c) = PART.captures(name.as_bytes()) { return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1; } test(&OLD_RAR, name) || test(&PAR2_VOL, name) } static TOOLS: LazyLock> = LazyLock::new(|| { vec![ ("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")), ("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")), ("ngPost", re(r"^[0-9a-f]{32}@ngPost$")), ("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")), ("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")), ("JBinUp", re(r"@JBin(?:Up|Down)\.local$")), ("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")), ("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")), ("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")), ] }); /// uploadTool with the regexes, as in the TypeScript (the reference for upload_tool's test). #[allow(dead_code)] pub fn upload_tool_re(message_id: &str) -> &'static str { if message_id.is_empty() { return "other"; } for (tool, pattern) in TOOLS.iter() { if test(pattern, message_id) { return tool; } } "other" } fn all(s: &[u8], f: impl Fn(u8) -> bool) -> bool { s.iter().all(|&c| f(c)) } fn lower_hex(c: u8) -> bool { c.is_ascii_digit() || (b'a'..=b'f').contains(&c) } /// `(?:[A-Z][a-z]){12}-\d{13}@`: nyuu's prefix (38 bytes, then '@'). fn nyuu_prefix(b: &[u8]) -> bool { b.len() > 38 && (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[24] == b'-' && all(&b[25..38], |c| c.is_ascii_digit()) && b[38] == b'@' } /// The posting tool of a Message-ID (TOOLS, in order, by hand: they ran for every per-article file). pub fn upload_tool(message_id: &str) -> &'static str { let b = message_id.as_bytes(); if b.is_empty() { return "other"; } if nyuu_prefix(b) && b.len() > 39 { // nyuu: "@nyuu" exactly; nyuu-host: any host (`.+`: no line terminators). if &b[39..] == b"nyuu" { return "nyuu"; } if !b[39..].iter().any(|&c| c == b'\n' || c == b'\r') && !message_id[39..].contains(['\u{2028}', '\u{2029}']) { return "nyuu-host"; } } let hex32 = b.len() > 33 && all(&b[..32], lower_hex) && b[32] == b'@'; if hex32 { let host = &b[33..]; if host == b"ngPost" { return "ngPost"; } if host.len() == 37 && all(&host[..32], lower_hex) && &host[32..] == b".yuun" { return "yuun"; } if all(host, |c| c.is_ascii_alphanumeric() || c == b'.' || c == b'-') { return "hex32"; } } if message_id.ends_with("@JBinUp.local") || message_id.ends_with("@JBinDown.local") { return "JBinUp"; } // (?i)^part\d+of\d+\. if b.len() > 4 && eq_ci(&b[..4], b"part") { let d1 = b[4..].iter().take_while(|c| c.is_ascii_digit()).count(); let of = 4 + d1; if d1 > 0 && b.len() > of + 2 && eq_ci(&b[of..of + 2], b"of") { let d2 = b[of + 2..].iter().take_while(|c| c.is_ascii_digit()).count(); if d2 > 0 && b.get(of + 2 + d2) == Some(&b'.') { return "partNofM"; } } } // ^\d{13}\$\d+\$\d+@localhost$ if let Some(rest) = message_id.strip_suffix("@localhost") { let r = rest.as_bytes(); let parts: Vec<&[u8]> = r.split(|&c| c == b'$').collect(); if parts.len() == 3 && parts[0].len() == 13 && parts.iter().all(|p| !p.is_empty() && all(p, |c| c.is_ascii_digit())) { return "ts-dollar"; } } // ^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$ if let Some(at) = b.iter().position(|&c| c == b'@') { let (local, host) = (&b[..at], &b[at + 1..]); if (10..=40).contains(&local.len()) && all(local, |c| c.is_ascii_alphanumeric()) { if let Some(dot) = host.iter().position(|&c| c == b'.') { let (name, tld) = (&host[..dot], &host[dot + 1..]); if (3..=12).contains(&name.len()) && (2..=6).contains(&tld.len()) && all(name, |c| c.is_ascii_alphanumeric()) && all(tld, |c| c.is_ascii_alphanumeric()) { return "random"; } } } } "other" } /// [A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,} anywhere. fn name_like(b: &[u8]) -> bool { let sep = |c: u8| matches!(c, b'.' | b'_' | b' ' | b'-'); (2..b.len()).any(|k| { if !(sep(b[k]) && b[k - 1].is_ascii_alphabetic() && b[k - 2].is_ascii_alphabetic()) { return false; } let m = k + b[k..].iter().take_while(|&&c| sep(c)).count(); m + 2 <= b.len() && b[m].is_ascii_alphanumeric() && b[m + 1].is_ascii_alphanumeric() }) } /// The second number of the first \[(\d+)/(\d+)\]. fn file_counter_total(s: &str) -> Option { let b = s.as_bytes(); for (i, &c) in b.iter().enumerate() { if c != b'[' { continue; } let a = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); let slash = i + 1 + a; if a == 0 || b.get(slash) != Some(&b'/') { continue; } let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); if d > 0 && b.get(slash + 1 + d) == Some(&b']') { return js::parse_int(&s[slash + 1..slash + 1 + d]); } } None } /// explored: a release in the sampled share of a rule's drops. pub fn explored(key: &str, percent: f64) -> bool { if percent <= 0.0 { return false; } let d = Md5::digest(key.as_bytes()); let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]); ((n % 10_000) as f64) < percent * 100.0 } static SIDECAR: LazyLock = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$")); static EXECUTABLE: LazyLock = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$")); pub struct JudgedFile<'a> { pub name: &'a str, pub total_parts: u32, pub groups: &'a [String], pub first_message_id: &'a str, } pub enum Verdict { Keep, Drop { rule: &'static str, key: String, sample: bool }, } /// The rules a lease brings (lib/rules Rules): what the worker drops. #[derive(Clone, Debug)] pub struct Rules { pub version: i64, /// Indexed groups and their priority (owner rule). pub priorities: rustc_hash::FxHashMap, /// "tool\tgroup" pub dropped_signatures: rustc_hash::FxHashSet, pub dropped_tools: rustc_hash::FxHashSet, pub per_article_max_parts: u32, pub sample_percent: f64, pub drop_executables: bool, } impl Rules { /// DEFAULT_RULES with these priorities and no dropped signatures (edge:bench's). pub fn defaults(priorities: rustc_hash::FxHashMap) -> Self { Rules { version: 1, priorities, dropped_signatures: Default::default(), dropped_tools: ["ngPost", "hex32", "yuun", "JBinUp", "random"].iter().map(|t| t.to_string()).collect(), per_article_max_parts: 3, sample_percent: 1.0, drop_executables: true, } } } /// RuleSet.judge. pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict { let sample = explored(release_key, rules.sample_percent); let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect(); if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) { let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap(); let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase(); return Verdict::Drop { rule: "executable", key, sample }; } if files.iter().any(|f| !is_random_file(f.name)) { return Verdict::Keep; } let first = &files[0]; let tool = upload_tool(first.first_message_id); let group = first.groups.first().map(String::as_str).unwrap_or(""); if rules.dropped_tools.contains(tool) || rules.dropped_signatures.contains(&format!("{tool}\t{group}")) { return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample }; } if files.iter().all(|f| f.total_parts <= rules.per_article_max_parts) { return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample }; } Verdict::Keep } #[cfg(test)] mod tests { use super::*; #[test] fn stems() { assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x"); assert_eq!(file_stem("x.part07.rar"), "x"); assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264"); assert_eq!(file_stem("a.7z.002"), "a"); } #[test] fn remainders() { assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release"); assert!(names_release("Some Release")); assert!(!names_release("elys de81bcabef5faf6a")); } #[test] fn by_hand_as_the_regexes() { let names = [ "x.mkv.vol03+04.par2", "x.part07.rar", "Movie.H.264.mkv.001", "a.7z.002", "a.7z", "b.zip.001", "B.ZIP", "c.r00", "c.r1234", "d.vol1+2.par2", "d.vol+2.par2", ".par2", "e.tar.gz", "f.mkv", "g.m3u8", "h.bin.001", " spaced.rar ", "i.PART01.RAR", "j.part.rar", "k.nfo.txt", "noext", "ünï.cödé.mkv.001", "l.ts.123", "m.r12.par2", "n.vol12-34.PAR2", "o.7z.1234", "p.123", "q.r1", ".rar.rar", ]; for n in names { assert_eq!(file_stem(n), file_stem_re(n), "{n}"); } let subjects = [ r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#, r#"[1/3] - "a.par2" yEnc (1/1) 123 "#, r#"yEnc "x" yenc yEncx (1/2)"#, r#"Show – S01E01 | 1080p : x "a" "b"#, r#"name (12/3x) [4/] "q" YENC 12 34"#, " lead\u{a0}nbsp yEnc\u{3000}99 ", r#"elys-de81bcabef5faf6a - [Nyh5Q07Y] - "x.mkv" yEnc"#, "a_yEnc b yEnc_c yEnc", "yEnc yEnc 5", "x yEnc 12 yEnc", "", ]; for s in subjects { assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}"); } } #[test] fn tools_by_hand_as_the_regexes() { let ids = [ "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@news.example", "b94268bc702040f7b811d5318f727e4f@ngPost", "b94268bc702040f7b811d5318f727e4f@b94268bc702040f7b811d5318f727e4f.yuun", "b94268bc702040f7b811d5318f727e4f@host-1.example", "B94268bc702040f7b811d5318f727e4f@ngPost", "x@JBinUp.local", "y@JBinDown.local", "part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local", "PART12OF3.x", "1790923856877$1729905757859$1@localhost", "179092385687$1$1@localhost", "t3Y4mV92YCp0McxDc5@zLZlsw.RYqes", "t3Y4mV92YC@zL.RYqes", "t3Y4mV92YCp0McxDc5@zLZlsw.RY.es", "", "a@b", "b94268bc702040f7b811d5318f727e4f@", "b94268bc702040f7b811d5318f727e4f@host_x", ]; for id in ids { assert_eq!(upload_tool(id), upload_tool_re(id), "{id}"); } for t in ["Some Release", "a b", "ab-cd", "ab - c1", "ab.c", "x ab_12", "12 ab", "ab "] { assert_eq!(name_like(t.as_bytes()), test(&NAME_LIKE, t), "{t}"); } for t in ["[01/45] x", "x [1/] [2/30]", "[a/1] [3/4]", "no counter"] { let re = FILE_COUNTER.captures(t.as_bytes()).and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap())); assert_eq!(file_counter_total(t), re, "{t}"); } } #[test] fn poster_tags_as_in_the_typescript() { let kill = "www.ggb.6x.to "; let a = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00"); let nfo = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo"); let b = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc"#, "Shnarph.-.Lautlos.2008[[g.6x.to]].r00"); assert_eq!(a.key, nfo.key); assert_ne!(a.key, b.key); assert_eq!(a.source, "stem"); let ftd1 = release_identity("p", r#"FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc"#, "Prison.Break.S04E02.720p.part01.rar"); let ftd2 = release_identity("p", r#"FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc"#, "Prison.Break.S04E02.720p.part.par2"); assert_eq!(ftd1.key, ftd2.key); let scene = release_identity("p", r#"Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc"#, "grp-sm.part01.rar"); assert_eq!(scene.name, "Some Movie 2008 DVDRip XviD GRP"); assert!(is_poster_tag("PowBeertje80 Posted", "Bravo Hits Vol.14 2CD -1996-")); assert!(!is_poster_tag("Audiofiles DIE ZEIT 35", "35_China_dl")); } #[test] fn sets_documents_and_audio_as_in_the_typescript() { let id = |subject: &str, name: &str| release_identity("p", subject, name); let a = id(r#"What about this? - File 027 of 132: "DSC_027.jpg" yEnc"#, "DSC_027.jpg"); let b = id(r#"What about this? - File 028 of 132: "DSC_028.jpg" yEnc"#, "DSC_028.jpg"); let other = id(r#"What about this? - File 001 of 90: "DSC_001.jpg" yEnc"#, "DSC_001.jpg"); assert_eq!(a.key, b.key); assert_ne!(a.key, other.key); assert_eq!(a.name, "What about this?"); let k1 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-76-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-76-lg.jpg"); let k2 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-77-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-77-lg.jpg"); let k3 = id(r#"Proud to be a poster "Office Chair set12-01-lg.jpg" yEnc"#, "Office Chair set12-01-lg.jpg"); assert_eq!(k1.key, k2.key); assert_ne!(k1.key, k3.key); for (subject, expected) in [ (r#"Minutemen (001/104) "x.cbr" - 76.88 MB - yEnc"#, "Minutemen"), (r#"Babes P A R T 3 ("Charlotte 03 16.jpg" 0097-1574 156.87 kB-273.03 MB) yEnc"#, "Babes P A R T 3"), (r#"*MaryKate* - OfficeChair [23of61] - "OfficeChair023.jpg" yEnc 187924 bytes"#, "*MaryKate* OfficeChair"), (r#"Adventure Comics [1948-12] - "x.rar""#, "Adventure Comics [1948 12]"), ] { assert_eq!(subject_remainder(subject), expected, "{subject}"); assert_eq!(subject_remainder_re(subject), expected, "{subject}"); } let c1 = id(r#"cw-horror&mystery [10/28] "Vampirella Magazine 061.cbr" yEnc"#, "Vampirella Magazine 061.cbr"); let c2 = id(r#"cw-horror&mystery [11/28] "Mystery Tales 12.cbr" yEnc"#, "Mystery Tales 12.cbr"); assert_ne!(c1.key, c2.key); assert_eq!(c2.name, "Mystery Tales 12"); let t1 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [04/59] - "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR" yEnc"#, "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR"); let t2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [34/59] - "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR" yEnc"#, "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR"); let par2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [59/59] - "Hörspiele Vol. 2.par2" yEnc"#, "Hörspiele Vol. 2.par2"); assert_eq!(t1.key, t2.key); assert_eq!(t1.key, par2.key); // A cover without words around it joins its stem's archive. assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key); } #[test] fn random_names() { assert!(looks_random("prvjr7hapd0i1ny14pz4")); assert!(looks_random("RmISakytFwXSCEzjVEpaKk")); assert!(looks_random("51294P1CUKR2R35X")); assert!(!looks_random("COD4MW2REMASTERED")); assert!(!looks_random("BEATLES1964REMASTER")); assert!(!looks_random("Some.Show.S01E01.1080p")); assert!(looks_random("VIDEO_TS")); } #[test] fn tools() { assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu"); assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost"); assert_eq!(upload_tool("x@y"), "other"); } }