//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools. //! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u` //! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS //! `\s` (which includes Unicode spaces) is spelled out as SP. use crate::js::{self, len16, sha1_hex}; use md5::{Digest, Md5}; use regex::bytes::Regex; use std::sync::LazyLock; /// JS `\s` as a byte pattern. const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)"; fn re(pattern: &str) -> Regex { Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap() } /// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all). fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String { let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) }; // The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8. String::from_utf8(out.into_owned()).unwrap() } fn test(re: &Regex, s: &str) -> bool { re.is_match(s.as_bytes()) } static SUFFIXES: LazyLock> = LazyLock::new(|| { vec![ // A part's par2 files ("X.part1.par2", "X.part1.vol00+34.par2"). (re(r"(?i)\.part[0-9]+(?:\.vol[0-9]+[+-][0-9]+)?\.par2$"), ""), (re(r"(?i)\.(?:part|par)(?:\.vol[0-9]+[+-][0-9]+)?\.par2$"), ""), (re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""), (re(r"(?i)\.par2$"), ""), (re(r"(?i)\.part[0-9]+\.(?:rar|rev)$"), ""), (re(r"(?i)\.r[0-9]{2,3}$"), ""), (re(r"(?i)\.(?:rar|rev)$"), ""), (re(r"(?i)(?:\.[0-9]{1,4})?\.bc$"), ""), (re(r"(?i)\.7z(\.[0-9]{3})?$"), ""), (re(r"(?i)\.zip(\.[0-9]{3})?$"), ""), // JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back. (re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|wmv|mov|mpg|mpeg|divx|webm|flv|m2v|mts|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"), (re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""), (re(r"(?i)\.tar$"), ""), (re(r"(?i)\.(zst|gz|xz|bz2)$"), ""), (re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|wmv|mov|mpg|mpeg)$"), ""), ] }); /// fileStem with the regexes, as in the TypeScript (the reference for file_stem's tests). #[allow(dead_code)] pub fn file_stem_re(name: &str) -> String { let trimmed = js::trim(name); let mut stem = if old_volume_at(trimmed.as_bytes()) { replace(&OLD_RAR_VOLUME, trimmed, "", false) } else { trimmed.to_string() }; let mut changed = true; while changed { changed = false; for (suffix, with) in SUFFIXES.iter() { if !suffix.is_match(stem.as_bytes()) { continue; } let next = replace(suffix, &stem, with, false); if next != stem && !next.is_empty() { stem = next; changed = true; } } } stem } /// Old-style rar volumes after .r99 and split zip volumes, as the file's own extension, lower case; .xNN but not /// .x32/.x64/.x86 (no lookahead here: the builds are put back by old_volume_at). static OLD_RAR_VOLUME: LazyLock = LazyLock::new(|| re(r"\.(?:[s-v][0-9]{2}|[xz][0-9]{2,3})$")); /// Whether a name ends with an old rar or split zip volume (OLD_RAR_VOLUME without the .x builds). fn old_volume_at(b: &[u8]) -> bool { let len = b.len(); let digits = b.iter().rev().take_while(|c| c.is_ascii_digit()).count(); if !(2..=3).contains(&digits) || len < digits + 2 || b[len - digits - 2] != b'.' { return false; } let letter = b[len - digits - 1]; let number = &b[len - digits..]; match letter { b's'..=b'v' => digits == 2, b'z' => true, // Not the builds .x32, .x64, .x86, .x264, .x265. b'x' => !matches!(number, b"32" | b"64" | b"86" | b"264" | b"265"), _ => false, } } static QUOTED: LazyLock = LazyLock::new(|| re(r#""[^"]*""#)); static COUNTERS: LazyLock = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)")); static YENC_END: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$")); static YENC: LazyLock = LazyLock::new(|| re(r"(?i)\byEnc\b")); static SEPARATORS: LazyLock = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+")); /// subjectRemainder with the regexes, as in the TypeScript (the reference for subject_remainder's tests). #[allow(dead_code)] pub fn subject_remainder_re(subject: &str) -> String { let s = replace("ED, subject, " ", true); let s = replace(&COUNTERS, &s, " ", true); let s = remove_counters_and_sizes(&s); let s = replace(&YENC_END, &s, " ", false); let s = replace(&YENC, &s, " ", true); let s = replace(&SEPARATORS, &s, " ", true); finish_remainder(&s) } fn eq_ci(a: &[u8], b: &[u8]) -> bool { a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_ascii_lowercase() == *y) } /// Ends in `suffix` (lowercase), ASCII case-insensitively. fn ends_ci(s: &[u8], suffix: &[u8]) -> bool { s.len() >= suffix.len() && eq_ci(&s[s.len() - suffix.len()..], suffix) } /// ASCII digits ending at `end`. fn digits_back(s: &[u8], end: usize) -> usize { s[..end].iter().rev().take_while(|c| c.is_ascii_digit()).count() } /// `.` + one of `exts` at the end: the length without it. fn strip_ext(s: &[u8], exts: &[&[u8]]) -> Option { for ext in exts { let n = ext.len() + 1; if s.len() >= n && s[s.len() - n] == b'.' && ends_ci(s, ext) { return Some(s.len() - n); } } None } /// `\.\d{3}$`: the length without it. fn strip_three_digits(s: &[u8]) -> Option { let n = s.len(); (n >= 4 && s[n - 4] == b'.' && s[n - 3..].iter().all(u8::is_ascii_digit)).then(|| n - 4) } /// \.\d{2,3}$: where a split volume's number starts (its dot). fn strip_volume_digits(s: &[u8]) -> Option { let n = s.len(); let d = s.iter().rev().take_while(|c| c.is_ascii_digit()).count(); ((2..=3).contains(&d) && n > d && s[n - d - 1] == b'.').then(|| n - d - 1) } const SPLIT_EXTS: [&[u8]; 23] = [ b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob", b"wmv", b"mov", b"mpg", b"mpeg", b"divx", b"webm", b"flv", b"m2v", b"mts", ]; const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"]; const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"]; const MEDIA_EXTS: [&[u8]; 16] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf", b"wmv", b"mov", b"mpg", b"mpeg"]; /// SUFFIXES[rule] once: the stem's new length when it matches. fn strip_suffix_rule(s: &[u8], rule: usize) -> Option { let n = s.len(); match rule { // \.vol\d+[+-]\d+\.par2$ 0 => { if !ends_ci(s, b".par2") { return None; } let d2 = digits_back(s, n - 5); let sign = (n - 5).checked_sub(d2 + 1).filter(|_| d2 > 0)?; if !matches!(s[sign], b'+' | b'-') { return None; } let d1 = digits_back(s, sign); let v = sign.checked_sub(d1 + 4).filter(|_| d1 > 0)?; eq_ci(&s[v..v + 4], b".vol").then_some(v) } 1 => ends_ci(s, b".par2").then(|| n - 5), // \.part\d+\.(?:rar|rev)$ 2 => { if !ends_ci(s, b".rar") && !ends_ci(s, b".rev") { return None; } let d = digits_back(s, n - 4); let p = (n - 4).checked_sub(d + 5).filter(|_| d > 0)?; eq_ci(&s[p..p + 5], b".part").then_some(p) } // \.r\d{2,3}$ 3 => { let d = digits_back(s, n); let p = n.checked_sub(d + 2).filter(|_| (2..=3).contains(&d))?; eq_ci(&s[p..p + 2], b".r").then_some(p) } 4 => (ends_ci(s, b".rar") || ends_ci(s, b".rev")).then(|| n - 4), // \.7z(\.\d{3})?$ and \.zip(\.\d{3})?$ 5 | 6 => { let ext: &[u8] = if rule == 5 { b".7z" } else { b".zip" }; if let Some(m) = strip_three_digits(s) { if ends_ci(&s[..m], ext) { return Some(m - ext.len()); } } ends_ci(s, ext).then(|| n - ext.len()) } // (?<=\.(7z|…))\.\d{2,3}$: only the volume number goes. 7 => { let m = strip_volume_digits(s)?; // Also after a rar volume ("rld-gtre.r13.002"): \.r\d{2,3} before it. strip_ext(&s[..m], &SPLIT_EXTS).or_else(|| strip_suffix_rule(&s[..m], 3)).map(|_| m) } 8 => strip_ext(s, &SIDE_EXTS), 9 => strip_ext(s, &[b"tar"]), 10 => strip_ext(s, &PACK_EXTS), 11 => strip_ext(s, &MEDIA_EXTS), // \.part\d+(?:\.vol\d+[+-]\d+)?\.par2$ 12 => { if !ends_ci(s, b".par2") { return None; } let m = strip_suffix_rule(s, 0).unwrap_or(n - 5); let d = digits_back(s, m); let p = m.checked_sub(d + 5).filter(|_| d > 0)?; eq_ci(&s[p..p + 5], b".part").then_some(p) } // \.(?:part|par)(?:\.vol\d+[+-]\d+)?\.par2$ 14 => { if !ends_ci(s, b".par2") { return None; } let m = strip_suffix_rule(s, 0).unwrap_or(n - 5); if m >= 5 && eq_ci(&s[m - 5..m], b".part") { Some(m - 5) } else if m >= 4 && eq_ci(&s[m - 4..m], b".par") { Some(m - 4) } else { None } } // (?:\.\d{1,4})?\.bc$ 13 => { if !ends_ci(s, b".bc") { return None; } let m = n - 3; let d = digits_back(s, m); Some(if (1..=4).contains(&d) && m > d && s[m - d - 1] == b'.' { m - d - 1 } else { m }) } _ => None, } } /// fileStem: per-file suffixes stripped, repeatedly. The SUFFIXES regexes by hand: with them, a /// file's description was most of the indexing time. pub fn file_stem(name: &str) -> String { let stem = js::trim(name); let b = stem.as_bytes(); let mut len = b.len(); // Old-style rar volumes after .r99 (".s41"), as the file's own extension, lower case. if old_volume_at(b) { len -= b.iter().rev().take_while(|c| c.is_ascii_digit()).count() + 2; } let mut changed = true; while changed { changed = false; // In the order of SUFFIXES: a part's par2 first, BCArchive volumes after the rar rules. for rule in [12, 14, 0, 1, 2, 3, 4, 13, 5, 6, 7, 8, 9, 10, 11] { if let Some(next) = strip_suffix_rule(&b[..len], rule) { if next > 0 && next < len { len = next; changed = true; } } } } // The suffixes are ASCII: `len` is a character boundary. stem[..len].to_string() } fn is_word(c: u8) -> bool { c.is_ascii_alphanumeric() || c == b'_' } /// "yEnc" (any case) at `i` with a word boundary (ASCII, as in JS) on both sides. fn yenc_at(b: &[u8], i: usize) -> bool { i + 4 <= b.len() && eq_ci(&b[i..i + 4], b"yenc") && (i == 0 || !is_word(b[i - 1])) && (i + 4 == b.len() || !is_word(b[i + 4])) } /// `(\s+\d+)?\s*$` fn yenc_tail(rest: &str) -> bool { let after_spaces = rest.trim_start_matches(js::is_space); if after_spaces.chars().all(js::is_space) { return true; } if after_spaces.len() == rest.len() { return false; } let after_digits = after_spaces.trim_start_matches(|c: char| c.is_ascii_digit()); after_digits.len() < after_spaces.len() && after_digits.chars().all(js::is_space) } /// subjectRemainder by hand: the same five replacements and trim. pub fn subject_remainder(subject: &str) -> String { let b = subject.as_bytes(); // "[^"]*" → ' ' let mut s1: Vec = Vec::with_capacity(b.len()); let mut i = 0; while i < b.len() { if b[i] == b'"' { if let Some(j) = b[i + 1..].iter().position(|&c| c == b'"') { s1.push(b' '); i += j + 2; continue; } } s1.push(b[i]); i += 1; } // \[\d+/\d+\] and \(\d+/\d+\) → ' ' let mut s2: Vec = Vec::with_capacity(s1.len()); let mut i = 0; while i < s1.len() { let close = match s1[i] { b'[' => Some(b']'), b'(' => Some(b')'), _ => None, }; if let Some(close) = close { let a = s1[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); let slash = i + 1 + a; if a > 0 && s1.get(slash) == Some(&b'/') { let d = s1[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); if d > 0 && s1.get(slash + 1 + d) == Some(&close) { s2.push(b' '); i = slash + 2 + d; continue; } } } s2.push(s1[i]); i += 1; } // Only ASCII was replaced, between whole characters. let mut s3 = remove_counters_and_sizes(&String::from_utf8(s2).unwrap()); // \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits) let b3 = s3.as_bytes(); if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) { s3.truncate(at); s3.push(' '); } // \byEnc\b → ' ', then runs of \s, -, –, ., :, | → ' ' let b3 = s3.as_bytes(); let mut s4 = String::with_capacity(s3.len()); let mut separator = false; let mut i = 0; while i < b3.len() { if yenc_at(b3, i) { // Its " " starts or joins a separator run. if !separator { s4.push(' '); separator = true; } i += 4; continue; } let c = s3[i..].chars().next().unwrap(); if js::is_space(c) || matches!(c, '-' | '.' | ':' | '|' | '\u{2013}') { if !separator { s4.push(' '); separator = true; } } else { s4.push(c); separator = false; } i += c.len_utf8(); } finish_remainder(&s4) } static OF_COUNTER: LazyLock = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b[\])]?")); static DASH_COUNTER: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)-([0-9]+)\]")); static SLASH_COUNTER: LazyLock = LazyLock::new(|| re(r"\b([0-9]+)/([0-9]+)\b")); static ZERO_COUNTER: LazyLock = LazyLock::new(|| re(r"\b(0[0-9]*)-([0-9]+)\b")); static SIZES: LazyLock = LazyLock::new(|| re(r"(?i)\b[0-9]+(?:[.,][0-9]+)?SP*(?:[kmgt]i?b|bytes?|byt)\b")); static BRACKET_SIZES: LazyLock = LazyLock::new(|| re(r"(?i)\[SP*[0-9]+(?:[.,][0-9]+)?SP*[kmgt]SP*\]")); static EMPTY_BRACKETS: LazyLock = LazyLock::new(|| re(r" ?(?:\[ ?\]|\( ?\)) ?")); fn number(digits: &[u8]) -> f64 { std::str::from_utf8(digits).unwrap().parse::().unwrap() } /// `s.replace(/…/g, counter)`: the match goes when its first number is at most its second. fn replace_counters(re: &Regex, s: &str) -> String { if !re.is_match(s.as_bytes()) { return s.to_string(); } let out = re.replace_all(s.as_bytes(), |c: ®ex::bytes::Captures| if number(&c[1]) <= number(&c[2]) { b" ".to_vec() } else { c[0].to_vec() }); String::from_utf8(out.into_owned()).unwrap() } /// The written-out counters, file sizes and empty brackets subjectRemainder removes after the [i/M] ones. fn remove_counters_and_sizes(s: &str) -> String { let s = replace_counters(&OF_COUNTER, s); let s = replace_counters(&DASH_COUNTER, &s); let s = replace_counters(&SLASH_COUNTER, &s); let s = replace_counters(&ZERO_COUNTER, &s); let s = replace(&SIZES, &s, " ", true); let s = replace(&BRACKET_SIZES, &s, " ", true); strip_number_before_yenc(&s) } /// `.replace(/\b\d+(\s+yEnc\s*)$/i, '$1')`: a size without a unit right before the closing yEnc. fn strip_number_before_yenc(s: &str) -> String { let tail_start = s.trim_end_matches(js::is_space).len(); let body = &s[..tail_start]; if body.len() < 4 || !body.is_char_boundary(body.len() - 4) || !body[body.len() - 4..].eq_ignore_ascii_case("yenc") { return s.to_string(); } let before = &body[..body.len() - 4]; let digits_end = before.trim_end_matches(js::is_space).len(); if digits_end == before.len() { return s.to_string(); // \s+ needs a space } let b = before.as_bytes(); let digits_start = b[..digits_end].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1); if digits_start == digits_end { return s.to_string(); } // \b: the digit run starts the text or follows a non-word character. if digits_start > 0 && (b[digits_start - 1].is_ascii_alphanumeric() || b[digits_start - 1] == b'_') { return s.to_string(); } // A short number after a word is a set's ("… set 1 [1 of 60] "destiny-001-001.jpg" yEnc"), not a size. let word_before = b[..digits_start].iter().rev().find(|&&c| !js::is_space(c as char)).is_some_and(|c| c.is_ascii_alphanumeric()); if digits_end - digits_start < 4 && word_before { return s.to_string(); } format!("{}{}", &s[..digits_start], &s[digits_end..]) } /// The end of subjectRemainder: what's left of brackets, double spaces, trim (after the separators became single spaces). fn finish_remainder(s: &str) -> String { let s = if s.contains(['[', '(']) { replace(&EMPTY_BRACKETS, s, " ", true) } else { s.to_string() }; let mut out = String::with_capacity(s.len()); for c in s.chars() { if c == ' ' && out.ends_with(' ') { continue; } out.push(c); } js::trim(&out).to_string() } /// displayName: a subject name as shown, brackets the counters left unmatched removed ("… Facialized 002/477] - …" /// gave "… Facialized ]"). Only the name: the key keeps the remainder. pub fn display_name(remainder: &str) -> String { let b = remainder.as_bytes(); if !b.iter().any(|c| matches!(c, b'[' | b']' | b'(' | b')')) { return remainder.to_string(); } let mut drop = vec![false; b.len()]; for (open, close) in [(b'[', b']'), (b'(', b')')] { let mut opens: Vec = Vec::new(); for (i, &c) in b.iter().enumerate() { if c == open { opens.push(i); } else if c == close && opens.pop().is_none() { drop[i] = true; } } for i in opens { drop[i] = true; } } if !drop.contains(&true) { return remainder.to_string(); } let kept: Vec = b.iter().zip(&drop).filter(|(_, &d)| !d).map(|(&c, _)| c).collect(); let kept = String::from_utf8(kept).expect("only ASCII brackets removed"); let mut out = String::with_capacity(kept.len()); for c in kept.chars() { if c == ' ' && out.ends_with(' ') { continue; } out.push(c); } let name = js::trim(&out); if name.is_empty() { remainder.to_string() } else { name.to_string() } } /// counterTotal: the subject's file counter total, first form found first. pub fn counter_total(subject: &str) -> Option { let s = replace("ED, subject, " ", true); if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) { let index = c.get(1).or(c.get(3)).unwrap().as_bytes(); let total = c.get(2).or(c.get(4)).unwrap().as_bytes(); return (!broken_total(index, total)).then(|| js_number_string(number(total))); } // Dates are no counters ("08-29-2011" read as 8 of 29). let undated = replace(&DATES, &s, " ", true); for re in [&*OF_COUNTER_OPEN, &*DASH_COUNTER, &*SLASH_COUNTER, &*ZERO_COUNTER] { for c in re.captures_iter(undated.as_bytes()) { if number(&c[1]) <= number(&c[2]) { return Some(js_number_string(number(&c[2]))); } } } None } static DATES: LazyLock = LazyLock::new(|| re(r"\b(?:(?:19|20)[0-9]{2}[-._ ](?:0?[1-9]|1[0-2])[-._ ][0-9]{1,2}|[0-9]{1,2}[-._ ][0-9]{1,2}[-._ ](?:19|20)[0-9]{2}|[0-9]{1,2}[-._](?:19|20)[0-9]{2})\b")); static COUNTERS_CAPTURE: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]|\(([0-9]+)/([0-9]+)\)")); /// The "of" counter without its closing bracket (as counterTotal's pattern). static OF_COUNTER_OPEN: LazyLock = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b")); /// `String(n)` of a counter's number (integers as JS prints them). fn js_number_string(n: f64) -> String { if n.fract() == 0.0 && n.abs() < 1e21 { format!("{}", n as i128) } else { format!("{n}") } } static IMAGE: LazyLock = LazyLock::new(|| re(r"(?i)\.(jpe?g|png|gif|bmp|webp|tiff?)$")); static AUDIO: LazyLock = LazyLock::new(|| re(r"(?i)\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])")); static DOCUMENT: LazyLock = LazyLock::new(|| re(r"(?i)\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$")); /// An image set's name without a counter: up to the last number, without numbers. /// imageSet (lib/release/identity.ts): the name up to the number that counts the images, digits kept. pub fn image_set(stem: &str) -> String { // .replace(/(\d+)\s*of\s*\d+/gi, '$1') // Suffixes that number no image: a copy's ("…[1]"), a resolution ("…_2000x1500"), a constant one ("…_00"). let mut s = stem.as_bytes().to_vec(); for re in [&*COPY_MARK, &*RESOLUTION, &*IMAGEVENUE, &*ZERO_SUFFIX] { s = re.replace(&s, &b""[..]).into_owned(); } // A counter in the name counts the images: the set is the text before it ("Set 008 10of63 new0464"). if let Some(m) = NAME_COUNTER.find(&s) { return String::from_utf8_lossy(&s[..m.start()]).into_owned(); } // A camera's timestamp ("kurozo-2006-7-26-15-21"): the date and the time after it number no set. let s_ts = TIMESTAMP.replace(&s, &b""[..]).into_owned(); s = s_ts; loop { // /\s*([_-])\s*(\d+)$/ (also spaced: "she has sensual legs 172 _ 172") let digits = s.iter().rev().take_while(|c| c.is_ascii_digit()).count(); if digits == 0 { break; } let ws = |c: &u8| matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c); let sep_end = s.len() - digits - s[..s.len() - digits].iter().rev().take_while(|c| ws(c)).count(); if sep_end == 0 || !matches!(s[sep_end - 1], b'_' | b'-') { break; } let sep = s[sep_end - 1]; let before_end = sep_end - 1 - s[..sep_end - 1].iter().rev().take_while(|c| ws(c)).count(); let n = &s[s.len() - digits..]; let before = &s[..before_end]; // /(\d+)\D*$/: the last digit run before let Some(last) = before.iter().rposition(u8::is_ascii_digit) else { break }; let start = before[..=last].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1); let previous = &before[start..=last]; // A repeat: the same number, or the end of the one before (054001 ends in 1). let same = trim_zeros(n) == trim_zeros(previous) || previous.ends_with(n); // A renumbering right after the number ("kate108lg029_28"): both count the images. let renumbered = sep == b'_' && n[0] != b'0' && last + 1 == before.len(); let year = n.len() == 4 && (n.starts_with(b"19") || n.starts_with(b"20")); if same || renumbered || year || n.len() >= 4 { let cut = before.len(); s.truncate(cut); } else { break; } } let set = match s.iter().rposition(u8::is_ascii_digit) { Some(last) => &s[..s[..=last].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1)], None => return String::new(), }; // A number of 5+ digits left in it is an id, per image ("sss-70556-3474182-23-10825418-pc"): no set. if set.split(|c| !c.is_ascii_digit()).any(|run| trim_zeros(run).len() >= 5) { return String::new(); } String::from_utf8_lossy(set).into_owned() } /// Number(digits) equality for digit strings: without leading zeros (all zeros: "0"). fn trim_zeros(d: &[u8]) -> &[u8] { let t = d.iter().position(|&c| c != b'0').map_or(&d[d.len().saturating_sub(1)..], |i| &d[i..]); t } static TIMESTAMP: LazyLock = LazyLock::new(|| re(r"[-_. ]?(?:19|20)[0-9]{2}[-._ ](?:0?[1-9]|1[0-2])[-._ ][0-9]{1,2}(?:[-._ ][0-9]{1,2})*$")); static COPY_MARK: LazyLock = LazyLock::new(|| re(r"(?:SP*[\[(][0-9]+[\])])+$")); static RESOLUTION: LazyLock = LazyLock::new(|| re(r"(?i)[_-][0-9]{3,4}(?:x[0-9]{3,4}|px)$")); static IMAGEVENUE: LazyLock = LazyLock::new(|| re(r"(?i)_123_[0-9]+lo$")); static ZERO_SUFFIX: LazyLock = LazyLock::new(|| re(r"[_-]0+$")); static NAME_COUNTER: LazyLock = LazyLock::new(|| re(r"(?i)([0-9]+)SP*ofSP*[0-9]+|[\[(]?SP*[0-9]+SP*/SP*[0-9]+SP*[\])]?")); fn image_prefix(stem: &str) -> String { let b = stem.as_bytes(); let cut = match b.iter().rposition(u8::is_ascii_digit) { Some(last) => b[..=last].iter().rposition(|c| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0), None => b.len(), }; let head = &stem[..cut]; let mut out = String::with_capacity(head.len()); let mut sep = false; for c in head.chars() { if c.is_ascii_digit() || js::is_space(c) || matches!(c, '.' | '_' | '-') { sep = true; } else { if sep && !out.is_empty() { out.push(' '); } out.push(c); sep = false; } } out } fn is_random_token(token: &str) -> bool { let t = token.strip_prefix('[').unwrap_or(token); let t = t.strip_suffix(']').unwrap_or(t); if len16(t) < 6 { return false; } let digit = t.bytes().any(|b| b.is_ascii_digit()); if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) { return true; } digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-')) } pub fn is_post_token(remainder: &str) -> bool { let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect(); let n = len16(&compact); (6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric()) } #[allow(dead_code)] static NAME_LIKE: LazyLock = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}")); pub fn names_release(remainder: &str) -> bool { let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect(); name_like(words.join(" ").as_bytes()) } #[derive(Debug, Clone)] pub struct Release { pub key: String, pub name: String, pub source: &'static str, /// The [i/M] total as parsed (0 is falsy in the TypeScript). pub file_total: Option, } impl Release { pub fn file_total_truthy(&self) -> Option { self.file_total.filter(|&n| n != 0) } } #[allow(dead_code)] static FILE_COUNTER: LazyLock = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]")); fn hash(parts: &[&str]) -> String { sha1_hex(&parts.join("\n")) } /// A name that says what it is: 12+ letters and digits, two words of 3+ letters (`describes`). fn describes(stem: &str) -> bool { let b = stem.as_bytes(); if b.iter().filter(|c| c.is_ascii_alphanumeric()).count() < 12 { return false; } // [A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,} let mut i = 0; while i < b.len() { let letters = b[i..].iter().take_while(|c| c.is_ascii_alphabetic()).count(); if letters == 0 { i += 1; continue; } let end = i + letters; if letters >= 3 { let gap = b[end..].iter().take_while(|c| !c.is_ascii_alphanumeric()).count(); if gap > 0 && b[end + gap..].iter().take_while(|c| c.is_ascii_alphabetic()).count() >= 3 { return true; } } i = end; } false } const TAG_WORDS: [&str; 21] = [ "www", "http", "https", "com", "net", "org", "info", "the", "and", "presents", "posted", "post", "powered", "yenc", "part", "rar", "par2", "nzb", "voor", "for", "from", ]; /// isPosterTag: the text outside the quotes shares no word with the file's stem. pub fn is_poster_tag(remainder: &str, stem: &str, counted: bool) -> bool { let lower = stem.to_lowercase(); let words: Vec<&str> = lower.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())).filter(|w| !w.is_empty()).collect(); let compact: String = words.concat(); let whole: std::collections::HashSet<&str> = words.iter().copied().collect(); // A per-post id, also inside a word ("FTD772544"), under a counter; without one the board's or the dump's. let b = remainder.as_bytes(); if counted && b.windows(5).any(|w| w.iter().all(u8::is_ascii_digit)) { return false; } let rem = remainder.to_lowercase(); let (mut significant, mut shared) = (0, 0); for word in rem.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())) { if !word.is_empty() && word.bytes().all(|c| c.is_ascii_digit()) { if word.len() >= 2 && whole.contains(word) { return false; } continue; } if word.len() < 3 || TAG_WORDS.contains(&word) { continue; } significant += 1; if if word.len() >= 4 { compact.contains(word) } else { whole.contains(word) } { shared += 1; } } // Without a counter, a tag of 4+ words may share one with the stem ("German Board - [Usenet4ever.info] und [SecretUsenet.com]"). // With one, that's a name ("[ MacGyver.S01E03.DVDRip.XviD-MEDiEVAL ]-[04/38] - "sample.macgy.s01e03-med.avi""). shared == 0 || (shared == 1 && significant >= 4 && !counted) } /// releaseIdentity for a named (not weak) file: never null. /// randomPoster (lib/release/identity.ts): a poster whose own name, before the "@", is a random token. pub fn random_poster(from: &str) -> bool { let address = match from.find('<').and_then(|open| from[open + 1..].find('>').map(|close| &from[open + 1..open + 1 + close])) { Some(address) => address, None => js::trim(from).split(' ').next().unwrap_or(""), }; // Separators don't make a token less random. let local: String = address.split_once('@').map_or(address, |(local, _)| local).chars().filter(|c| !matches!(c, '-' | '_' | '.')).collect(); looks_random(&local) } /// RUN_ON (lib/release/identity.ts): an archive's or par2's extension. static RUN_ON: LazyLock = LazyLock::new(|| re(r"(?i)\.(?:part[0-9]+\.rar|vol[0-9]+[+-][0-9]+\.par2|par2|rar|r[0-9]{2,3}|7z(?:\.[0-9]{3})?|zip|nzb|nfo|sfv)")); /// runOn (lib/release/identity.ts): a quoted name that runs on past its extension into a title, cut there. pub fn run_on(name: &str) -> &str { let b = name.as_bytes(); let ascii = b.len() >= 4 && b[..4].is_ascii(); for m in RUN_ON.find_iter(b) { let rest = &name[m.end()..]; if rest.is_empty() { return name; } // Not a bare dash ("….READ.NFO-THETA.rar"). let spaced = rest.starts_with(js::is_space); let repeats = ascii && rest.len() >= 4 && rest.as_bytes()[..4].eq_ignore_ascii_case(&b[..4]); if spaced || repeats { return &name[..m.end()]; } } name } /// sizeAfterName (lib/release/identity.ts): the subject without a number after the file's unquoted name at its end. pub fn size_after_name(subject: &str, name: &str) -> String { let end = subject.trim_end_matches(js::is_space); let before_digits = end.trim_end_matches(|c: char| c.is_ascii_digit()); if before_digits.len() == end.len() { return subject.to_string(); } // /\s*-\s*$/ let t = before_digits.trim_end_matches(js::is_space); let before = match t.strip_suffix('-') { Some(t) => t.trim_end_matches(js::is_space), None => return subject.to_string(), }; if !name.is_empty() && before.ends_with(name) && !before.ends_with(&format!("\"{name}\"")) { before.to_string() } else { subject.to_string() } } /// A name posted unquoted with its size in bytes right after it, at the subject's end ("… Nude Celebrity Pictures <<<<< - /// !index0001.jpg 75220 bytes"): a tool that posts a folder under the subject typed for it (sizedName). Not a quoted name. pub fn sized_name(subject: &str, name: &str) -> bool { let s = subject.trim_end(); let lower = s.to_ascii_lowercase(); let unit = if lower.ends_with("bytes") { 5 } else if lower.ends_with("byte") { 4 } else { return false }; let s = s[..s.len() - unit].trim_end(); // /\d[\d.,]*$/: from the first digit of the trailing run of digits, dots and commas. let run = s.len() - s.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.' || c == ',').len(); let tail = &s[s.len() - run..]; let Some(first) = tail.find(|c: char| c.is_ascii_digit()) else { return false }; let s = s[..s.len() - run + first].trim_end(); let s = s.strip_suffix('-').unwrap_or(s).trim_end(); !name.is_empty() && s.ends_with(name) && !s.ends_with(&format!("\"{name}\"")) } pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release { let poster = if random_poster(poster) { "" } else { poster }; let sized = sized_name(subject, name); // The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal). let post_total = file_counter_total(subject); let stem = random_volume_base(file_stem(run_on(name))); // A name that isn't quoted (unquoted_name) is taken out as the quotes would have been. // The subject here has its spaces collapsed; the name may not. // Other quoted text stays: it names the release (">>> CREATIVE COMMONS NZB <<< "0190 alarm 5.0 " - File 1 of 9: …"). // A size after a name that isn't quoted, at the end ("[27/70] - 2407.jpg - 28"). let subject = size_after_name(subject, name); let subject = subject.as_str(); let quoted = format!("\"{name}\""); let without = if subject.contains("ed) { subject.replacen("ed, " ", 1) } else { let collapsed = js::collapse_spaces(name); let collapsed = js::trim(&collapsed); if subject.contains(name) { subject.replacen(name, " ", 1) } else { subject.replacen(collapsed, " ", 1) } }; // The name repeated elsewhere ("(NELET11.par2) [01/58] - "NELET11.par2""). let without = if name.encode_utf16().count() >= 5 { without.replace(name, " ") } else { without }; let named = without.replace('"', " "); // A file's size in brackets after its counter ("[26/32] - [4929] "…part24.rar""). let named = replace(&SIZE_AFTER_COUNTER, &named, "$1", true); // A per-file hex id in brackets ("… [1f1793f68d5321] [newzNZB] [01/22]"). let named = replace(&HEX_ID, &named, " ", true); // Counted without the file name: "shyla-shy-09-0195.jpg hardcore …" is no "9 of 195". let total = counter_total(&named); let file_total = total.as_ref().and_then(|t| t.parse::().ok()); // Without a counter a host and a sponsor name no upload ("(www.Thunder-News.org) >XXX< "). let remainder = subject_remainder(&named); let remainder = if total.is_none() { let bare = replace(&BOILERPLATE, &remainder, " ", true); js::trim(&js::collapse_spaces(&bare)).to_string() } else { remainder }; // A document or comic is a release of its own. if test(&DOCUMENT, name) { let shown = replace(&DOCUMENT, &stem, "", false); let name = if shown.is_empty() { stem.clone() } else { shown }; return Release { key: hash(&["stem", poster, &stem]), name, source: "stem", file_total }; } let image = test(&IMAGE, name); // The poster's constant tag outside the quotes: the file's stem says which upload it is (not for images and audio). // Not for numbered items of a set ("003 Defining Tools with JSON Schema.mp4"): /^\d{1,3}[ ._-]/. // /^(?:[A-Z]{1,5}\d{2,6}[-_ ])?\d{1,3}[ ._-]/: also after a disc code ("SC8367-12 - …"). let numbered = { let b = stem.as_bytes(); let index_at = |at: usize| { let digits = b[at..].iter().take_while(|c| c.is_ascii_digit()).count(); (1..=3).contains(&digits) && matches!(b.get(at + digits), Some(b' ' | b'.' | b'_' | b'-')) }; let letters = b.iter().take_while(|c| c.is_ascii_uppercase()).count(); let code_digits = b[letters..].iter().take_while(|c| c.is_ascii_digit()).count(); let code = (1..=5).contains(&letters) && (2..=6).contains(&code_digits) && matches!(b.get(letters + code_digits), Some(b'-' | b'_' | b' ')); let date = date_led(b); (index_at(0) || (code && index_at(letters + code_digits + 1))) && !date }; // A hashed stem under text without a counter is the upload, the text a tag ("Seit 1753 Tagen - usenet4ever.info - …"). let hashed = file_total.is_none() && test(&HASHED, &stem); // A dated stem says which upload it is, short as it is ("09-30-2008_CTC.r00"). let dated = date_led(stem.as_bytes()); let tag = hashed || (!image && !numbered && !test(&AUDIO, name) && (describes(&stem) || dated) && !looks_random(&stem) && is_poster_tag(&remainder, &stem, total.is_some())); let prefix = if image { image_prefix(&stem) } else { String::new() }; // An image set's subject may be one word, the model's name: /^[A-Za-z]{3,}$/. let one_word = image && remainder.len() >= 3 && remainder.bytes().all(|c| c.is_ascii_alphabetic()); if !tag && (names_release(&remainder) || one_word) { // Without a counter an image's set: its name up to its last number, digits kept; none without numbers. // A folder's images posted with their sizes: the subject names the folder, the images no set. let bound = total.clone().unwrap_or_else(|| if image && !sized { image_set(&stem) } else { String::new() }); let mut parts: Vec<&str> = vec!["subject", poster, &remainder, &bound]; // An upload's random stem on its volumes ("Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar" under "[ TOWN ] … [ TV ]"). let stripped: String = stem.chars().filter(|c| !matches!(c, '-' | '_')).collect(); if !image && test(&RANDOM_VOLUME, name) && stem.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-') && looks_random(&stripped) { parts.push(&stem); } return Release { key: hash(&parts), name: display_name(&remainder), source: "subject", file_total }; } if let Some(total) = post_total.filter(|&n| n != 0 && !tag) { if is_post_token(&remainder) { return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: display_name(&remainder), source: "post", file_total }; } } // A scene music post's index files ("00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg") under a bare counter go // with its tracks: the artist and the total. if let Some(total) = post_total.filter(|&n| n != 0) { if stem.len() > 2 && stem.starts_with("00") && matches!(stem.as_bytes()[2], b'-' | b'_' | b' ' | b'.') { if let Some(c) = SCENE_TRACK.captures(stem.as_bytes()) { let artist = String::from_utf8(c[1].to_vec()).unwrap(); return Release { key: hash(&["stem", poster, &artist, &total.to_string()]), name: artist, source: "stem", file_total }; } } // And its par2 and sfv under the release's own name ("blaise_mc-in_hellstill_3-ep-web-2022-known.par2"). // Only a track post's lower-case name: a rar upload groups by its full name. if !test(&AUDIO, name) && !stem.chars().any(|c| c.is_ascii_uppercase() || js::is_space(c)) { if let Some(c) = SCENE_RELEASE.captures(stem.as_bytes()) { let artist = String::from_utf8(c[1].to_vec()).unwrap(); return Release { key: hash(&["stem", poster, &artist, &total.to_string()]), name: artist, source: "stem", file_total }; } } } if image { // Keyed by its set when its name has one ("diana-040-001.jpg"). let name = if prefix.is_empty() { stem.clone() } else { prefix.clone() }; let set = image_set(&stem); let set = if set.is_empty() { name.clone() } else { set }; // Under a post token ("MDG005b - (01/90) "IMG_2307.JPG""): the post's too. // A word in it, not a size ("… alexis001.jpg 57295b"). let post = is_post_token(&remainder) && remainder.as_bytes().windows(2).any(|w| w[0].is_ascii_alphabetic() && w[1].is_ascii_alphabetic()); // Under a counter, bounded by its total ("lop1001.jpg", "pic004.jpg": every post of the poster one release). let mut parts: Vec<&str> = vec!["stem", poster, &set]; if post { parts.push(&remainder); } // Not a cover named as its archive's stem ("x.jpg" beside "x.part01.rar"). if let Some(t) = total.as_deref().filter(|_| set != stem) { parts.push(t); } let key = hash(&parts); return Release { key, name, source: "stem", file_total }; } // An audiobook's or album's tracks without a name around them: by their name without its numbers. if test(&AUDIO, name) { // Scene tracks under a bare counter ("02-repiet_-_dont_think_twice_(extended_mix).mp3"): the artist's album. if let Some(total) = post_total.filter(|&n| n != 0) { if let Some(c) = SCENE_TRACK.captures(stem.as_bytes()) { let artist = String::from_utf8(c[1].to_vec()).unwrap(); return Release { key: hash(&["stem", poster, &artist, &total.to_string()]), name: artist, source: "stem", file_total }; } } let tracks = image_prefix(&stem); if !tracks.is_empty() { // Bounded by a counter's total too ("r86_022 - Gwen Guthrie - … .mp3" → "r"). let key = match total.as_deref() { Some(t) => hash(&["stem", poster, &tracks, t]), None => hash(&["stem", poster, &tracks]), }; return Release { key, name: tracks, source: "stem", file_total }; } } // A season's episodes under a bare counter ("Pokemon - S02E01 - Pallet Party Panic.mkv"): per show, season and total. if let Some(total) = post_total.filter(|&n| n != 0) { if test(&VIDEO, name) { if let Some(c) = EPISODE.captures(stem.as_bytes()) { let show = String::from_utf8(c[1].to_vec()).unwrap(); let season = format!("{} S{:0>2}", js::trim(&show), std::str::from_utf8(&c[2]).unwrap()); return Release { key: hash(&["stem", poster, &season.to_lowercase(), &total.to_string()]), name: season, source: "stem", file_total }; } } } // A numbered volume as the extension, within the counter's total ("DN.VPL PCK.1508" of 2195): one upload. // Not after a digit: "CHT PCK 2 9.26.001" looks just like a dump's versions ("K-ML 3.8.233"). let volume = VOLUME_NUMBER.captures(stem.as_bytes()); let base = match (volume, post_total.filter(|&n| n != 0)) { (Some(c), Some(total)) if number(&c[2]) <= total as f64 => String::from_utf8(c[1].to_vec()).unwrap(), _ => stem.clone(), }; // Without case and a scene index's "00-" ("00-laura_van_dam_-_…-talion.m3u" beside "Laura_van_Dam_-_…-TALiON.par2"). let lower = base.to_lowercase(); let b = lower.as_bytes(); let index = if b.starts_with(b"00") { 2 + b[2..].iter().take_while(|c| matches!(c, b'-' | b'_' | b' ' | b'.')).count() } else { 0 }; let key_stem = if index > 2 { &lower[index..] } else { &lower[..] }; // Nor spaces at a dash ("… XVID- Farbiges Krabbeltier.r00" beside "… XVID-Farbiges Krabbeltier.vol03+04.par2"). let key_stem = replace(&DASH_SPACES, key_stem, "-", true); Release { key: hash(&["stem", poster, &key_stem]), name: stem, source: "stem", file_total } } static DASH_SPACES: LazyLock = LazyLock::new(|| re(r" *- *")); static HEX_ID: LazyLock = LazyLock::new(|| re(r"(?i)\[[0-9a-f]{8,}\]")); static VIDEO: LazyLock = LazyLock::new(|| re(r"(?i)\.(mkv|mp4|avi|m4v|ts|wmv|mov)$")); static EPISODE: LazyLock = LazyLock::new(|| re(r"(?i)^(.+?)[ ._-]+S([0-9]{1,2})E[0-9]{1,3}")); static SIZE_AFTER_COUNTER: LazyLock = LazyLock::new(|| re(r"(\[[0-9]+/[0-9]+\])SP*-SP*\[[0-9]+\]")); /// A host and a sponsor in a subject's remainder (BOILERPLATE in lib/release/identity.ts). static BOILERPLATE: LazyLock = LazyLock::new(|| re(r"(?i)\(?\bwww(?: [A-Za-z0-9]+)*? (?:org|com|net|info|nl|de|eu|to|biz|us)\b\)?|?")); /// Also one long hex run ("6c57319fbae505679737bd46a76193a1-u4e.part6.rar"). static HASHED: LazyLock = LazyLock::new(|| re(r"(?i)^(?:[0-9a-f]{8,}(?:_[0-9a-f]{6,}){2,}|[0-9a-f]{24,})(?:-[0-9a-z]+)?$")); static SCENE_TRACK: LazyLock = LazyLock::new(|| re(r"^[0-9]{1,3}[-_ .]+(.+?)(?:_-_| - |-)")); /// SCENE_RELEASE (lib/release/identity.ts): a scene music release's name. static SCENE_RELEASE: LazyLock = LazyLock::new(|| { re(r"(?i)^(.+?)(?:_-_| - |-).*-(?:web|cd[a-z]?|vinyl|vls|sat|cable|fm|dab|single|ep|lp|promo|bootleg)-.*(?:19|20)[0-9]{2}(?:-[A-Za-z0-9_]+)?$") }); /// RANDOM_VOLUME (lib/release/identity.ts): a file of a multi-volume upload. static RANDOM_VOLUME: LazyLock = LazyLock::new(|| re(r"(?i)\.(?:part[0-9]+\.rar|vol[0-9]+[+-][0-9]+\.par2|par2|r[0-9]{2,3}|7z\.[0-9]{3}|zip\.[0-9]{3}|[0-9]{3})$")); /// RANDOM_PART (lib/release/identity.ts): a random name's volume with any short extension, or none. static RANDOM_PART: LazyLock = LazyLock::new(|| re(r"^(.+)\.part[0-9]{2,}(?:\.[A-Za-z0-9]{1,5})?$")); /// randomVolumeBase (lib/release/identity.ts): a random name's volume number and extension off its stem. pub fn random_volume_base(stem: String) -> String { let base = RANDOM_PART.captures(stem.as_bytes()).map(|c| String::from_utf8(c[1].to_vec()).unwrap()); match base { Some(b) if b.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-') && looks_random(&b.chars().filter(|c| !matches!(c, '-' | '_')).collect::()) => b, _ => stem, } } static VOLUME_NUMBER: LazyLock = LazyLock::new(|| re(r"^(.*[^0-9. ])[. ]([0-9]{1,5})\.?$")); static GENERIC: LazyLock = LazyLock::new(|| { re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$") }); fn is_generic_name(name: &str) -> bool { test(&GENERIC, js::trim(name)) } /// looksRandom. pub fn looks_random(name: &str) -> bool { // name.replace(/\.[A-Za-z0-9]{1,4}$/, '') let name = match name.rfind('.') { Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i], _ => name, }; if is_generic_name(name) { return true; } let b = name.as_bytes(); let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit()); let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit()); if b.len() >= 16 && hex(b) { return true; } if let Some(dot) = name.find('.') { let (h, d) = (&b[..dot], &b[dot + 1..]); if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) { return true; } } if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) { return true; } { let core = name.strip_suffix(" n").unwrap_or(name).as_bytes(); if core.len() >= 6 && digits(core) { return true; } } let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count(); let n_digit = count(u8::is_ascii_digit); let n_lower = count(u8::is_ascii_lowercase); let n_upper = count(u8::is_ascii_uppercase); let alnum = b.iter().all(|c| c.is_ascii_alphanumeric()); if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 { return true; } if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 { return true; } // 51294P1CUKR2R35X: capitals and digits only, switching between them all over the token // (COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice). if b.len() >= 12 && b.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) && b.windows(2).filter(|w| w[0].is_ascii_digit() != w[1].is_ascii_digit()).count() >= 6 { return true; } if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 { return false; } n_digit > 0 || n_upper >= 6 } pub fn is_random_file(name: &str) -> bool { looks_random(&file_stem(name)) } /// Readable or not (lib/release/readable.ts): whether a name says what a release is, read token by token, so /// separators, spaces and random "extensions" don't hide a scramble that looks_random's single-token shapes miss /// (LOG.md (129)). The readable flag; the drop rules keep looks_random, which is narrower. /// Tokens that don't make a name readable on their own: packaging, volume numbers and the posting tools' fixed words. static NOISE_TOKEN: LazyLock = LazyLock::new(|| re(r"(?i)^(addon|samples?|proof|subs?|nfo|part[0-9]+|vol[0-9]+|description|repost|rep|extras?|bonus|cd[0-9]*|dvd[0-9]*|dis[ck][0-9]*|par2|rar|zip|7z|mkv|mp4|avi|vol)$")); /// Mixed tokens that are codes, not scrambles: S02E10, 1080p, 10bit, x264, Win10x64, BDRip720p, 3DMark03, v2, 5th. static CODE_TOKEN: LazyLock = LazyLock::new(|| re(r"^([sS][0-9]{1,2}[eE][0-9]{1,3}|[xh]26[45]|[A-Za-z]+[0-9]+([a-z]{1,3}|x(64|86))?|[0-9]+[A-Za-z]{1,3}|[0-9]+[A-Za-z]+[0-9]+)$")); /// The NZBSPLIT tools' hash between their markers is no part of the name. static NZBSPLIT_HASH: LazyLock = LazyLock::new(|| re(r"(?i)__NZBS[P]LIT__[0-9a-f]{32}__NZBS[P]LIT__")); // "SP" is re()'s space class /// An inner run of 2+ capitals with lowercase on both sides ("tBWTuz", "YDoaROv"; not "3DMark03"). static INNER_UPPER: LazyLock = LazyLock::new(|| re(r"[a-z][A-Z]{2,}[a-z]")); /// Scene tags write "i" for "I" between capitals (XCiTE, PRESCRiPTiON): read it as a capital there. fn scene_case(t: &[u8]) -> Vec { let mut out = t.to_vec(); for i in 1..t.len().saturating_sub(1) { if t[i] == b'i' && t[i - 1].is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() { out[i] = b'I'; } } out } /// The signals of a scrambled token (readable.ts signals()). struct TokenSignals { /// Letter/digit and lower-to-upper switches. switches: usize, /// Case switches in both directions. case_switches: usize, /// Lone lowercase letters (not "i") between capitals ("NtpUgCmS"). lone: usize, inner_upper: bool, max_consonants: usize, /// Vowels, a lowercase y too ("McCartney"). vowels: usize, /// Runs of one character class. runs: usize, letters: usize, /// The longest lowercase run. max_lower: usize, } fn token_signals(t: &[u8]) -> TokenSignals { let is_vowel = |c: u8| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u') || c == b'y'; let class = |c: u8| if c.is_ascii_digit() { 0 } else if c.is_ascii_lowercase() { 1 } else { 2 }; let mut s = TokenSignals { switches: 0, case_switches: 0, lone: 0, inner_upper: false, max_consonants: 0, vowels: 0, runs: 0, letters: 0, max_lower: 0 }; let mut consonants = 0; let mut lower = 0; for i in 0..t.len() { let c = t[i]; if c.is_ascii_alphabetic() && !is_vowel(c) && c.to_ascii_lowercase() != b'y' { consonants += 1; s.max_consonants = s.max_consonants.max(consonants); } else { consonants = 0; } if is_vowel(c) { s.vowels += 1; } if c.is_ascii_alphabetic() { s.letters += 1; } if c.is_ascii_lowercase() { lower += 1; s.max_lower = s.max_lower.max(lower); } else { lower = 0; } if i == 0 || class(t[i - 1]) != class(c) { s.runs += 1; } if i == 0 { continue; } let a = t[i - 1]; if a.is_ascii_digit() != c.is_ascii_digit() { s.switches += 1; } else if a.is_ascii_lowercase() && c.is_ascii_uppercase() { s.switches += 1; } if (a.is_ascii_lowercase() && c.is_ascii_uppercase()) || (a.is_ascii_uppercase() && c.is_ascii_lowercase()) { s.case_switches += 1; } if i < t.len() - 1 && c.is_ascii_lowercase() && c != b'i' && a.is_ascii_uppercase() && t[i + 1].is_ascii_uppercase() { s.lone += 1; } } s.inner_upper = INNER_UPPER.is_match(t); s } #[derive(PartialEq, Eq, Debug, Clone, Copy)] pub enum TokenKind { Word, Random, Neutral, } /// What one alphanumeric token of a name is: a word (letters only, in one case or Capitalized, 3+ long, not a /// packaging word), a scramble, or neither (digits, codes, CamelCase, short tokens). Kept in step with readable.ts. pub fn token_kind(t: &[u8]) -> TokenKind { let n = t.len(); if t.iter().all(u8::is_ascii_digit) || NOISE_TOKEN.is_match(t) || CODE_TOKEN.is_match(t) { return TokenKind::Neutral; } if t.iter().all(u8::is_ascii_alphabetic) { let s = token_signals(&scene_case(t)); let capitalized = t[0].is_ascii_uppercase() && t[1..].iter().all(u8::is_ascii_lowercase); if s.case_switches == 0 || (s.case_switches == 1 && capitalized) { // A base32 consonant soup ("mrrtmzrzmi4dgmrzmyytqmlbztczbx" without its digit) is no word. if n >= 10 && s.vowels * 6 < n { return TokenKind::Random; } return if n >= 3 { TokenKind::Word } else { TokenKind::Neutral }; } // Mixed case beyond Capitalization: CamelCase (neutral) or a scramble, which has few vowels and short runs. // Two CamelCase words with few vowels (FastFilms, BestCrypt, SchiffsModell) have four runs and no lone // letter; a scramble has more runs, a lone letter or an inner capital run. if n >= 6 && s.case_switches >= 2 && s.vowels * 4 < n && (s.runs >= 5 || s.lone >= 1 || s.inner_upper) { return TokenKind::Random; } return TokenKind::Neutral; } if n >= 8 && t.iter().all(u8::is_ascii_hexdigit) && t.iter().filter(|c| c.is_ascii_digit()).count() >= 2 { return TokenKind::Random; } if t.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) { // 51294P1CUKR2R35X; COD4MW2REMASTERED (4 switches) and BEATLES1964REMASTER (2) stay. return if n >= 12 && token_signals(t).switches >= 6 { TokenKind::Random } else { TokenKind::Neutral }; } // Words with numbers between them (Laura15Juli2026, Back4WinXP): every letter run a word or an acronym. if t.split(|c: &u8| !c.is_ascii_alphabetic()).filter(|w| !w.is_empty()).all(|w| { let vowels = w.iter().filter(|c| matches!(c.to_ascii_lowercase(), b'a' | b'e' | b'i' | b'o' | b'u' | b'y')).count(); (w.len() >= 2 && w.iter().all(u8::is_ascii_uppercase)) || (w.len() >= 2 && w[1..].iter().all(u8::is_ascii_lowercase) && (w.len() >= 3 || w[0].is_ascii_lowercase()) && vowels * 5 >= w.len()) }) { return TokenKind::Neutral; } let s = token_signals(&scene_case(t)); if n >= 5 && s.switches >= 3 { return TokenKind::Random; } if n >= 6 && s.switches >= 2 && (s.lone >= 1 || s.inner_upper || s.max_consonants >= 4) { return TokenKind::Random; } // Few vowels and no lowercase run of 3+ ("Jk5Pud", "RKi9BCg"; not "HDR10Plus"). if n >= 6 && s.switches >= 2 && s.vowels * 4 < s.letters && s.max_lower <= 2 { return TokenKind::Random; } TokenKind::Neutral } /// A file or release name no one can read (readable.ts isUnreadableName): looks_random's shapes, or more scrambled /// tokens than words by letter count ("Zx5AjNaK-mh7CFg6Hf8uu", "Description - Lt5nHjlo3RiuBb3C", "PsNdnYiugfZ"; not /// "Paul McCartney - Band on the Run", "ShesSoWild - Ester Ana 1080p"). pub fn is_unreadable_name(name: &str) -> bool { let stem = file_stem(name); let stem = NZBSPLIT_HASH.replace(stem.as_bytes(), &b" "[..]); if looks_random(&String::from_utf8_lossy(&stem)) { return true; } let mut words = 0; let mut random = 0; for t in stem.split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) { match token_kind(t) { TokenKind::Word => words += t.len(), TokenKind::Random => random += t.len(), TokenKind::Neutral => {} } } random > 0 && random > words } /// The class of an unnamed release (lib/release/classes.ts pendingClass; LOG.md (130)): size bucket × name shape × /// partial, 1..70. The lease's dropped classes ("tool\tgroup\tklass") are skipped at the edge. pub fn pending_class(name: &str, bytes: u64, partial: bool) -> u8 { let size = [1_000_000u64, 10_000_000, 100_000_000, 1_000_000_000].iter().filter(|&&b| bytes >= b).count() as u8; let s = match name.rfind('.') { Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i], _ => name, }; let b = s.as_bytes(); let all = |f: fn(&u8) -> bool| !b.is_empty() && b.iter().all(f); let shape: u8 = if b.len() >= 16 && all(u8::is_ascii_hexdigit) { 1 } else if b.len() >= 6 && all(u8::is_ascii_digit) { 2 } else if b.len() >= 20 && all(|c| c.is_ascii_lowercase() || (b'2'..=b'7').contains(c)) && b.iter().any(|c| (b'2'..=b'7').contains(c)) { 3 } else if all(u8::is_ascii_alphabetic) { 4 } else if all(u8::is_ascii_alphanumeric) { 5 } else if all(|c| c.is_ascii_alphanumeric() || matches!(c, b'.' | b'_' | b'-')) { 6 } else { 7 }; size * 14 + (shape - 1) * 2 + u8::from(partial) + 1 } /// A class as droppedClasses lists it. pub fn class_key(tool: &str, group: &str, klass: u8) -> String { format!("{tool}\t{group}\t{klass}") } /// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case /// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"). /// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators. pub fn looks_scrambled(name: &str) -> bool { let stem = file_stem(name); let (mut total, mut scrambled) = (0, 0); for token in stem.as_bytes().split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) { total += token.len(); let switches = token.windows(2).filter(|w| (w[0].is_ascii_lowercase() && w[1].is_ascii_uppercase()) || w[0].is_ascii_digit() != w[1].is_ascii_digit()).count(); if token.len() >= 5 && switches >= 3 { scrambled += token.len(); } } total >= 8 && scrambled * 2 >= total } static SEVEN_SPLIT: LazyLock = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$")); static SEVEN: LazyLock = LazyLock::new(|| re(r"(?i)\.7z$")); static PART1: LazyLock = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$")); static RAR: LazyLock = LazyLock::new(|| re(r"(?i)\.rar$")); static PART: LazyLock = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$")); static BIN: LazyLock = LazyLock::new(|| re(r"(?i)\.(bin|zip)$")); static OLD_RAR: LazyLock = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$")); static PAR2_VOL: LazyLock = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$")); pub static PAR2: LazyLock = LazyLock::new(|| re(r"(?i)\.par2$")); pub fn is_par2(name: &str) -> bool { test(&PAR2, name) } pub fn seven_zip_volume(name: &str) -> Option { if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) { return js::parse_int(std::str::from_utf8(&c[1]).unwrap()); } if test(&SEVEN, name) { Some(0) } else { None } } pub fn is_archive_start(name: &str) -> bool { if test(&BIN, name) { return true; } if matches!(seven_zip_volume(name), Some(0) | Some(1)) { return true; } test(&PART1, name) || (test(&RAR, name) && !test(&PART, name)) } pub fn is_later_volume(name: &str) -> bool { if let Some(c) = PART.captures(name.as_bytes()) { return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1; } test(&OLD_RAR, name) || test(&PAR2_VOL, name) } static TOOLS: LazyLock> = LazyLock::new(|| { vec![ ("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")), ("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")), ("ngPost", re(r"^[0-9a-f]{32}@ngPost$")), ("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")), ("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")), ("JBinUp", re(r"@JBin(?:Up|Down)\.local$")), ("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")), ("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")), ("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")), ] }); /// uploadTool with the regexes, as in the TypeScript (the reference for upload_tool's test). #[allow(dead_code)] pub fn upload_tool_re(message_id: &str) -> &'static str { if message_id.is_empty() { return "other"; } for (tool, pattern) in TOOLS.iter() { if test(pattern, message_id) { return tool; } } "other" } fn all(s: &[u8], f: impl Fn(u8) -> bool) -> bool { s.iter().all(|&c| f(c)) } fn lower_hex(c: u8) -> bool { c.is_ascii_digit() || (b'a'..=b'f').contains(&c) } /// `(?:[A-Z][a-z]){12}-\d{13}@`: nyuu's prefix (38 bytes, then '@'). fn nyuu_prefix(b: &[u8]) -> bool { b.len() > 38 && (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[24] == b'-' && all(&b[25..38], |c| c.is_ascii_digit()) && b[38] == b'@' } /// The posting tool of a Message-ID (TOOLS, in order, by hand: they ran for every per-article file). pub fn upload_tool(message_id: &str) -> &'static str { let b = message_id.as_bytes(); if b.is_empty() { return "other"; } if nyuu_prefix(b) && b.len() > 39 { // nyuu: "@nyuu" exactly; nyuu-host: any host (`.+`: no line terminators). if &b[39..] == b"nyuu" { return "nyuu"; } if !b[39..].iter().any(|&c| c == b'\n' || c == b'\r') && !message_id[39..].contains(['\u{2028}', '\u{2029}']) { return "nyuu-host"; } } let hex32 = b.len() > 33 && all(&b[..32], lower_hex) && b[32] == b'@'; if hex32 { let host = &b[33..]; if host == b"ngPost" { return "ngPost"; } if host.len() == 37 && all(&host[..32], lower_hex) && &host[32..] == b".yuun" { return "yuun"; } if all(host, |c| c.is_ascii_alphanumeric() || c == b'.' || c == b'-') { return "hex32"; } } if message_id.ends_with("@JBinUp.local") || message_id.ends_with("@JBinDown.local") { return "JBinUp"; } // (?i)^part\d+of\d+\. if b.len() > 4 && eq_ci(&b[..4], b"part") { let d1 = b[4..].iter().take_while(|c| c.is_ascii_digit()).count(); let of = 4 + d1; if d1 > 0 && b.len() > of + 2 && eq_ci(&b[of..of + 2], b"of") { let d2 = b[of + 2..].iter().take_while(|c| c.is_ascii_digit()).count(); if d2 > 0 && b.get(of + 2 + d2) == Some(&b'.') { return "partNofM"; } } } // ^\d{13}\$\d+\$\d+@localhost$ if let Some(rest) = message_id.strip_suffix("@localhost") { let r = rest.as_bytes(); let parts: Vec<&[u8]> = r.split(|&c| c == b'$').collect(); if parts.len() == 3 && parts[0].len() == 13 && parts.iter().all(|p| !p.is_empty() && all(p, |c| c.is_ascii_digit())) { return "ts-dollar"; } } // ^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$ if let Some(at) = b.iter().position(|&c| c == b'@') { let (local, host) = (&b[..at], &b[at + 1..]); if (10..=40).contains(&local.len()) && all(local, |c| c.is_ascii_alphanumeric()) { if let Some(dot) = host.iter().position(|&c| c == b'.') { let (name, tld) = (&host[..dot], &host[dot + 1..]); if (3..=12).contains(&name.len()) && (2..=6).contains(&tld.len()) && all(name, |c| c.is_ascii_alphanumeric()) && all(tld, |c| c.is_ascii_alphanumeric()) { return "random"; } } } } "other" } /// [A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,} anywhere. fn name_like(b: &[u8]) -> bool { let sep = |c: u8| matches!(c, b'.' | b'_' | b' ' | b'-' | b',' | b'&' | b':' | b';' | b'\\' | b'/' | b'+'); (2..b.len()).any(|k| { if !(sep(b[k]) && b[k - 1].is_ascii_alphabetic() && b[k - 2].is_ascii_alphabetic()) { return false; } let m = k + b[k..].iter().take_while(|&&c| sep(c)).count(); m + 2 <= b.len() && b[m].is_ascii_alphanumeric() && b[m + 1].is_ascii_alphanumeric() }) } /// /^\d{1,2}[-._]\d{1,2}[-._](?:19|20)\d{2}(?!\d)/: a stem that starts with a date ("09-30-2008_BAK_JRS"). fn date_led(b: &[u8]) -> bool { let run = |at: usize| b.get(at..).map_or(0, |s| s.iter().take_while(|c| c.is_ascii_digit()).count()); let sep = |at: usize| matches!(b.get(at), Some(b'-' | b'.' | b'_')); let d1 = run(0); let d2 = if (1..=2).contains(&d1) && sep(d1) { run(d1 + 1) } else { 0 }; let y = d1 + 1 + d2 + 1; (1..=2).contains(&d2) && sep(d1 + 1 + d2) && run(y) == 4 && matches!(&b[y..y + 2], b"19" | b"20") } /// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal). fn broken_total(index: &[u8], total: &[u8]) -> bool { total.len() > 1 && total[0] == b'0' && total.len() > index.len() } /// The second number of the first \[(\d+)/(\d+)\]. fn file_counter_total(s: &str) -> Option { let b = s.as_bytes(); for (i, &c) in b.iter().enumerate() { if c != b'[' { continue; } let a = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); let slash = i + 1 + a; if a == 0 || b.get(slash) != Some(&b'/') { continue; } let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); if d > 0 && b.get(slash + 1 + d) == Some(&b']') { if broken_total(&b[i + 1..slash], &b[slash + 1..slash + 1 + d]) { return None; } return js::parse_int(&s[slash + 1..slash + 1 + d]); } } None } /// explored: a release in the sampled share of a rule's drops. pub fn explored(key: &str, percent: f64) -> bool { if percent <= 0.0 { return false; } let d = Md5::digest(key.as_bytes()); let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]); ((n % 10_000) as f64) < percent * 100.0 } static SIDECAR: LazyLock = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$")); /// A sidecar by its extension (the judge's SIDECAR): no release of its own, and no album's member. pub fn is_sidecar(name: &str) -> bool { test(&SIDECAR, name) } static EXECUTABLE: LazyLock = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$")); /// An unsplit archive (the judge's ARCHIVE and VOLUME): one zip, rar or 7z that is not a volume of a set. static ARCHIVE: LazyLock = LazyLock::new(|| re(r"(?i)\.(zip|rar|7z)$")); /// "cneg" is "part" in ROT13: the a.b.erotica posts named that way are volumes of passworded videos (LOG.md (128)). static VOLUME: LazyLock = LazyLock::new(|| re(r"(?i)\.((part|cneg)[0-9]+\.rar|r[0-9][0-9]|z[0-9][0-9]|7z\.[0-9][0-9][0-9]|[0-9][0-9][0-9])$")); /// A poster at a free mail provider (the judge's WEBMAIL): the stub floods' throwaway accounts. static WEBMAIL: LazyLock = LazyLock::new(|| { re(r"(?i)@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\.") }); /// A name that reads as a release (the judge's RELEASE_LIKE): a year, a scene tag, a group suffix or dotted words. static RELEASE_LIKE: LazyLock = LazyLock::new(|| { re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)") }); /// RELEASE_LIKE with spaces as separators too: the tiny fakes are posted under names written either way. static RELEASE_LIKE_SPACED: LazyLock = LazyLock::new(|| { re(r"(?i)(\b(19|20)[0-9][0-9]\b|[._ -](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s[0-9][0-9]e[0-9][0-9]|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v[0-9]+(\.[0-9]+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)") }); /// The stub-archive rule (LOG.md (123)-(126), (128)): one unsplit archive, no par2, settled, posted to 2+ groups, /// under STUB_MAX_BYTES (the fakes stop at ~90 MB; the hash-named uploads, real albums and passworded videos above /// it don't), and its articles at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES (Power-Post's /// 250 kB default; a one-article release always has small articles, so e-books and sidecars are exempt), or 4+ /// groups, or a webmail poster. pub const STUB_ARTICLE_BYTES: u64 = 262_144; pub const STUB_MIN_BYTES: u64 = 1_000_000; pub const STUB_MAX_BYTES: u64 = 100_000_000; pub const STUB_WIDE_GROUPS: usize = 4; /// The nonsense-piece rule: one article holding one archive of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a /// release-like name that isn't scrambled (the newznzb.* pieces; under 100 kB are cue/log sidecars and adverts). pub const NONSENSE_MIN_BYTES: u64 = 100_000; pub const NONSENSE_MAX_BYTES: u64 = 1_000_000; /// The tiny-archive rule's floor: a RAR holding one exe stub is 2 kB and up; below are empty archives and nfo rars. pub const TINY_MIN_BYTES: u64 = 2_000; pub struct JudgedFile<'a> { pub name: &'a str, pub total_parts: u32, pub groups: &'a [String], pub first_message_id: &'a str, pub poster: &'a str, /// Articles seen and their bytes (the overview's :bytes). pub segments: u32, pub bytes: u64, } /// One unsplit archive and nothing else. fn single_archive<'a, 'b>(files: &'b [JudgedFile<'a>]) -> Option<&'b JudgedFile<'a>> { match files { [f] if test(&ARCHIVE, f.name) && !test(&VOLUME, f.name) => Some(f), _ => None, } } /// Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way /// the stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the /// stub-archive rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)). static STUB_EXEMPT_GROUPS: LazyLock = LazyLock::new(|| re(r"(?i)(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)")); /// The legs of the stub-archive rule a settled single archive under STUB_MAX_BYTES trips ("A" article size, "X" 4+ /// groups, "W" webmail), empty when none. pub fn stub_archive_legs(f: &JudgedFile) -> String { let mut legs = String::new(); if f.groups.len() < 2 || f.bytes >= STUB_MAX_BYTES || f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) { return legs; } let article = f.bytes / u64::from(f.segments.max(1)); if article <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES { legs.push('A'); } if f.groups.len() >= STUB_WIDE_GROUPS { legs.push('X'); } if test(&WEBMAIL, f.poster) { legs.push('W'); } legs } /// An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece. static EBOOK: LazyLock = LazyLock::new(|| re(r"(?i)\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b")); /// A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the /// newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces). pub fn is_nonsense_piece(f: &JudgedFile) -> bool { f.total_parts == 1 && f.segments == 1 && (NONSENSE_MIN_BYTES..NONSENSE_MAX_BYTES).contains(&f.bytes) && !f.name.contains(' ') && test(&RELEASE_LIKE, &file_stem(f.name)) && !test(&EBOOK, f.name) && !looks_scrambled(f.name) && !is_random_file(f.name) } /// Words of the real tiny archives under release-like names: subtitles, cracks, trainers and keygens (a few kB of /// exe is the content), covers and sidecars. static TINY_CONTENT: LazyLock = LazyLock::new(|| { re(r"(?i)\b(subs?|subtitles?|subpack|srt|untertitel|ondertitels?|ondertiteling|legendas?|napisy|sottotitoli|cracks?|cracked|keygen|keymaker|trainer|patch|patched|serials?|activator|loader|nocd|no-cd|fix|covers?|artwork|booklet|scans?|proof|sample|info|readme)\b") }); /// A tiny archive: one article of TINY_MIN_BYTES to NONSENSE_MIN_BYTES holding one archive under a release-like, /// unscrambled name (spaces allowed), outside the groups that post real small archives, and without an e-book, /// subtitle, crack or sidecar word: the 2015-17 exe-stub flood in miniature (LOG.md (131)). pub fn is_tiny_archive(f: &JudgedFile) -> bool { f.total_parts == 1 && f.segments == 1 && (TINY_MIN_BYTES..NONSENSE_MIN_BYTES).contains(&f.bytes) && !f.groups.iter().any(|g| test(&STUB_EXEMPT_GROUPS, g)) && test(&RELEASE_LIKE_SPACED, &file_stem(f.name)) && !test(&EBOOK, f.name) && !test(&TINY_CONTENT, f.name) && !looks_scrambled(f.name) && !is_random_file(f.name) } pub enum Verdict { Keep, Drop { rule: &'static str, key: String, sample: bool }, } /// The rules a lease brings (lib/rules Rules): what the worker drops. #[derive(Clone, Debug)] pub struct Rules { pub version: i64, /// Indexed groups and their priority (owner rule). pub priorities: rustc_hash::FxHashMap, /// "tool\tgroup" pub dropped_signatures: rustc_hash::FxHashSet, /// "tool\tgroup\tklass" (pending_class): unnamed releases of these classes don't go to the pending store. pub dropped_classes: rustc_hash::FxHashSet, pub dropped_tools: rustc_hash::FxHashSet, pub per_article_max_parts: u32, pub sample_percent: f64, pub drop_executables: bool, /// Fragments of the poster lowercased without punctuation (droppedPosters). pub dropped_posters: Vec, /// The stub-archive rule (dropStubArchives), the nonsense-piece rule (dropNonsensePieces) and the tiny-archive /// rule (dropTinyArchives). pub drop_stub_archives: bool, pub drop_nonsense_pieces: bool, pub drop_tiny_archives: bool, } impl Rules { /// DEFAULT_RULES with these priorities and no dropped signatures (edge:bench's). pub fn defaults(priorities: rustc_hash::FxHashMap) -> Self { Rules { version: 1, priorities, dropped_signatures: Default::default(), dropped_classes: Default::default(), dropped_tools: ["ngPost", "hex32", "yuun", "JBinUp", "random"].iter().map(|t| t.to_string()).collect(), per_article_max_parts: 3, sample_percent: 1.0, drop_executables: true, dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(), drop_stub_archives: true, drop_nonsense_pieces: true, drop_tiny_archives: true, } } } /// posterWords: lowercased, letters and digits only. pub fn poster_words(poster: &str) -> String { poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect() } /// RuleSet.judge. `settled`: the window closed on the release (a par2 can still join an open one), which the /// single-archive rules need. pub fn judge(release_key: &str, files: &[JudgedFile], settled: bool, rules: &Rules) -> Verdict { let sample = explored(release_key, rules.sample_percent); if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) { let words = poster_words(first.poster); if let Some(dropped) = rules.dropped_posters.iter().find(|p| words.contains(p.as_str())) { return Verdict::Drop { rule: "poster", key: dropped.clone(), sample }; } } let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect(); if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) { let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap(); let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase(); return Verdict::Drop { rule: "executable", key, sample }; } if let (true, Some(f)) = (settled, single_archive(files)) { if rules.drop_stub_archives { let legs = stub_archive_legs(f); if !legs.is_empty() { return Verdict::Drop { rule: "stub-archive", key: legs, sample }; } } if rules.drop_nonsense_pieces && is_nonsense_piece(f) { return Verdict::Drop { rule: "nonsense-piece", key: upload_tool(f.first_message_id).to_string(), sample }; } if rules.drop_tiny_archives && is_tiny_archive(f) { return Verdict::Drop { rule: "tiny-archive", key: upload_tool(f.first_message_id).to_string(), sample }; } } if files.iter().any(|f| !is_random_file(f.name)) { return Verdict::Keep; } let first = &files[0]; let tool = upload_tool(first.first_message_id); let group = first.groups.first().map(String::as_str).unwrap_or(""); if rules.dropped_tools.contains(tool) || rules.dropped_signatures.contains(&format!("{tool}\t{group}")) { return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample }; } if files.iter().all(|f| f.total_parts <= rules.per_article_max_parts) { return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample }; } Verdict::Keep } #[cfg(test)] mod tests { use super::*; #[test] fn stems() { assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x"); assert_eq!(file_stem("x.part07.rar"), "x"); assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264"); assert_eq!(file_stem("a.7z.002"), "a"); assert_eq!(file_stem("Morning Lovers.wmv.058"), "Morning Lovers"); assert_eq!(file_stem("clip.mov.12"), "clip"); assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers"); } #[test] fn scrambled_names() { for n in ["nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa", "PNSVWSV-xLaNv6o5qS840e3lwSZyH", "Biq-YsXu8G33ktt"] { assert!(looks_scrambled(n), "{n}"); } // Short ones of the spam too (8% of alt.binaries.storage's): too few switches to tell. for n in ["A1Monitor v6.2.2", "PDF2Word v1.3", "Some.Show.S01E01.1080p.WEB.part1.rar", "MacGyver.S01E01.mkv", "Jimmy and the Crawler (mobi)", "Q25m.PDUYtbA", "82WH.2iuTK"] { assert!(!looks_scrambled(n), "{n}"); } } #[test] fn display_names() { let porn4you = "webmaster@hdbangvidz.com (Porn4You)"; let a = release_identity(porn4you, "Babe with Plump Bubble Butt Fucked and Facialized 002/477] - 0178.jpg", "0178.jpg"); let b = release_identity(porn4you, "Babe with Plump Bubble Butt Fucked and Facialized 003/477] - 0179.jpg", "0179.jpg"); assert_eq!(a.name, "Babe with Plump Bubble Butt Fucked and Facialized"); assert_eq!(a.key, b.key); assert_eq!(display_name("[Group] Show (2020)"), "[Group] Show (2020)"); assert_eq!(display_name("( Show [x] name"), "Show [x] name"); assert_eq!(display_name("]"), "]"); } #[test] fn remainders() { assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release"); assert!(names_release("Some Release")); assert!(!names_release("elys de81bcabef5faf6a")); } #[test] fn by_hand_as_the_regexes() { let names = [ "x.mkv.vol03+04.par2", "x.part07.rar", "Movie.H.264.mkv.001", "a.7z.002", "a.7z", "b.zip.001", "B.ZIP", "c.r00", "c.r1234", "d.vol1+2.par2", "d.vol+2.par2", ".par2", "e.tar.gz", "f.mkv", "g.m3u8", "h.bin.001", " spaced.rar ", "i.PART01.RAR", "j.part.rar", "k.nfo.txt", "noext", "ünï.cödé.mkv.001", "l.ts.123", "m.r12.par2", "n.vol12-34.PAR2", "o.7z.1234", "p.123", "q.r1", ".rar.rar", ]; for n in names { assert_eq!(file_stem(n), file_stem_re(n), "{n}"); } let subjects = [ r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#, r#"[1/3] - "a.par2" yEnc (1/1) 123 "#, r#"yEnc "x" yenc yEncx (1/2)"#, r#"Show – S01E01 | 1080p : x "a" "b"#, r#"name (12/3x) [4/] "q" YENC 12 34"#, " lead\u{a0}nbsp yEnc\u{3000}99 ", r#"elys-de81bcabef5faf6a - [Nyh5Q07Y] - "x.mkv" yEnc"#, "a_yEnc b yEnc_c yEnc", "yEnc yEnc 5", "x yEnc 12 yEnc", "", ]; for s in subjects { assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}"); } } #[test] fn tools_by_hand_as_the_regexes() { let ids = [ "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@news.example", "b94268bc702040f7b811d5318f727e4f@ngPost", "b94268bc702040f7b811d5318f727e4f@b94268bc702040f7b811d5318f727e4f.yuun", "b94268bc702040f7b811d5318f727e4f@host-1.example", "B94268bc702040f7b811d5318f727e4f@ngPost", "x@JBinUp.local", "y@JBinDown.local", "part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local", "PART12OF3.x", "1790923856877$1729905757859$1@localhost", "179092385687$1$1@localhost", "t3Y4mV92YCp0McxDc5@zLZlsw.RYqes", "t3Y4mV92YC@zL.RYqes", "t3Y4mV92YCp0McxDc5@zLZlsw.RY.es", "", "a@b", "b94268bc702040f7b811d5318f727e4f@", "b94268bc702040f7b811d5318f727e4f@host_x", ]; for id in ids { assert_eq!(upload_tool(id), upload_tool_re(id), "{id}"); } for t in ["Some Release", "a b", "ab-cd", "ab - c1", "ab.c", "x ab_12", "12 ab", "ab "] { assert_eq!(name_like(t.as_bytes()), test(&NAME_LIKE, t), "{t}"); } for t in ["[01/45] x", "x [1/] [2/30]", "[a/1] [3/4]", "no counter"] { let re = FILE_COUNTER.captures(t.as_bytes()).and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap())); assert_eq!(file_counter_total(t), re, "{t}"); } } #[test] fn poster_tags_as_in_the_typescript() { let kill = "www.ggb.6x.to "; let a = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00"); let nfo = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo"); let b = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc"#, "Shnarph.-.Lautlos.2008[[g.6x.to]].r00"); assert_eq!(a.key, nfo.key); assert_ne!(a.key, b.key); assert_eq!(a.source, "stem"); let ftd1 = release_identity("p", r#"FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc"#, "Prison.Break.S04E02.720p.part01.rar"); let ftd2 = release_identity("p", r#"FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc"#, "Prison.Break.S04E02.720p.part.par2"); let kere = |n: &str| release_identity("p", &format!(" - ID - 1855891249 - {n}"), n); assert_ne!(kere("Along.The.Ridge.2006.Limited.PAL.DVDr-TWCiSO.rar").key, kere("Altera.QUARTUS.II.DSP.Builder.v7.2.SP1-SHooTERS.rar").key); assert_eq!(ftd1.key, ftd2.key); let scene = release_identity("p", r#"Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc"#, "grp-sm.part01.rar"); assert_eq!(scene.name, "Some Movie 2008 DVDRip XviD GRP"); assert!(is_poster_tag("PowBeertje80 Posted", "Bravo Hits Vol.14 2CD -1996-", false)); assert!(!is_poster_tag("Audiofiles DIE ZEIT 35", "35_China_dl", false)); } #[test] fn sets_documents_and_audio_as_in_the_typescript() { let id = |subject: &str, name: &str| release_identity("p", subject, name); let a = id(r#"What about this? - File 027 of 132: "DSC_027.jpg" yEnc"#, "DSC_027.jpg"); let b = id(r#"What about this? - File 028 of 132: "DSC_028.jpg" yEnc"#, "DSC_028.jpg"); let other = id(r#"What about this? - File 001 of 90: "DSC_001.jpg" yEnc"#, "DSC_001.jpg"); assert_eq!(a.key, b.key); assert_ne!(a.key, other.key); assert_eq!(a.name, "What about this?"); let k1 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-76-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-76-lg.jpg"); let k2 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-77-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-77-lg.jpg"); let k3 = id(r#"Proud to be a poster "Office Chair set12-01-lg.jpg" yEnc"#, "Office Chair set12-01-lg.jpg"); assert_eq!(k1.key, k2.key); assert_ne!(k1.key, k3.key); for (subject, expected) in [ (r#"Minutemen (001/104) "x.cbr" - 76.88 MB - yEnc"#, "Minutemen"), (r#"Babes P A R T 3 ("Charlotte 03 16.jpg" 0097-1574 156.87 kB-273.03 MB) yEnc"#, "Babes P A R T 3"), (r#"*MaryKate* - OfficeChair [23of61] - "OfficeChair023.jpg" yEnc 187924 bytes"#, "*MaryKate* OfficeChair"), (r#"Adventure Comics [1948-12] - "x.rar""#, "Adventure Comics [1948 12]"), ] { assert_eq!(subject_remainder(subject), expected, "{subject}"); assert_eq!(subject_remainder_re(subject), expected, "{subject}"); } let c1 = id(r#"cw-horror&mystery [10/28] "Vampirella Magazine 061.cbr" yEnc"#, "Vampirella Magazine 061.cbr"); let c2 = id(r#"cw-horror&mystery [11/28] "Mystery Tales 12.cbr" yEnc"#, "Mystery Tales 12.cbr"); assert_ne!(c1.key, c2.key); assert_eq!(c2.name, "Mystery Tales 12"); let t1 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [04/59] - "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR" yEnc"#, "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR"); let t2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [34/59] - "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR" yEnc"#, "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR"); let par2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [59/59] - "Hörspiele Vol. 2.par2" yEnc"#, "Hörspiele Vol. 2.par2"); assert_eq!(t1.key, t2.key); assert_eq!(t1.key, par2.key); // A cover without words around it joins its stem's archive. assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key); } #[test] fn pending_classes() { use super::pending_class; // Size bucket × shape × partial: a 5 MB hex name whole, the same partial, a 500 MB scramble with separators. assert_eq!(pending_class("c04212a389cf69edc54767355ab5285f", 5_000_000, false), 15); assert_eq!(pending_class("c04212a389cf69edc54767355ab5285f", 5_000_000, true), 16); assert_eq!(pending_class("Zx5AjNaK-mh7CFg6Hf8uu", 500_000_000, false), 3 * 14 + 5 * 2 + 1); assert_eq!(pending_class("1650777167", 100, false), 3); assert_eq!(pending_class("mrrtmzrzmi4dgmrzmyytqmlbztczbx", 2_000_000_000, false), 4 * 14 + 2 * 2 + 1); assert_eq!(pending_class("PsNdnYiugfZ.rar", 20_000_000, false), 2 * 14 + 3 * 2 + 1); assert_eq!(pending_class("n5d1ueIf34X", 20_000_000, true), 2 * 14 + 4 * 2 + 2); assert_eq!(pending_class("sgkra vU7u2yuw9LDLCjDu04pOez", 999, false), 13); } #[test] fn unreadable_names() { use super::is_unreadable_name; // Scrambles with separators, spaces, a random "extension", a packaging word, a hex with an addon (LOG.md (129)). for n in [ "Zx5AjNaK-mh7CFg6Hf8uu.part01.rar", "yWjfnT2ujG4wJj3fG46o0FLzsgkvhmrTsAfW2.E0WsOU", "PsNdnYiugfZ.rar", "phfplC9y07U2yrj", "n5d1ueIf34X", "tJqr1tq7xK", "Description - Lt5nHjlo3RiuBb3C.par2", "bd5b4a59f3aa87114dcf75c0-addon.rar", "mrrtmzrzmi4dgmrzmyytqmlbztczbx", "AA-NaR16302830434zxP32108309f3Ab", "_QBs01_MqZYnYwM_iAyRquhc-", "sgkra vU7u2yuw9LDLCjDu04pOez", "Jk5Pud.zQdNj", "7uKsD_NRgT-G_Uc_p8", "NtpUgCmS_Xa1B2Pk", "{E11BF8E9-E2C9-4DAE-A1B2-8027BC465F3F}", "IkYovjrtZrItjQ.mkv", "ff75ce3f16c746c18b2d78cf.6c52f", ] { assert!(is_unreadable_name(n), "{n}"); } // Real names: scene tags, CamelCase, codes, leet, an acronym, a hash beside words. for n in [ "Dark.Matter.2024.S02E10.No.Quiet.Life.1080p.ATVP.WEB-DL.DDP5.1.Atmos.H.264-Kitsune", "Paul McCartney - Band on the Run", "ShesSoWild - Ester Ana 1080p", "MyDadsHotGirlfriend.13.02.01.Luna.Star.XXX.1080p.x264-SEXORS", "cFosSpeed 11.0 Build 2420 x86 x64 incl Patch.rar", "Rocketship-Hurt_Feelings-WEB-2026-PreMaxXxing", "Drum-and-BassDJ_Samurai-Sound_With_Fire_Artemis-(FQY026)-WEB-2006-PRESCRiPTiON", "Kleverig-H4rd-N12uk3b18-luc16037-720p-smpwr", "Eclipse_IDE_2023-12_R_win32_x86_64.zip", "PartsHPGL 1.06c", "Go4Customer Fax Machine v1.2b", "AdBeGone 1.2", "3DMark03 320", "COD4MW2REMASTERED", "Movie.2020.1080p-GRP [a3f9e2b1]", "EPL__NZBSPLIT__f1fdb433772a1a7771d6b73c875009ff__NZBSPLIT__EPL.2026.Liverpool.vs.Brighton.1080pEN50fps", "iPhone", "Some.Show.S01E01.1080p.mkv", "FastFilms 2.04", "BestCrypt NP v4.0", "EasyHTML 2.35 German", "SchiffsModell - September 2026", "CrossStitcher - Issue 440, October 2026", "Laura15Juli2026", "IHaveAWife - Isa Bella", "DickHDDaily - Santana Red", "1904-SmithsWork-Hasluck", ] { assert!(!is_unreadable_name(n), "{n}"); } } #[test] fn random_names() { assert!(looks_random("prvjr7hapd0i1ny14pz4")); assert!(looks_random("RmISakytFwXSCEzjVEpaKk")); assert!(looks_random("51294P1CUKR2R35X")); assert!(!looks_random("COD4MW2REMASTERED")); assert!(!looks_random("BEATLES1964REMASTER")); assert!(!looks_random("Some.Show.S01E01.1080p")); assert!(looks_random("VIDEO_TS")); } #[test] fn tools() { assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu"); assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost"); assert_eq!(upload_tool("x@y"), "other"); } #[cfg(test)] mod numbered_tests { use super::release_identity; #[test] fn numbered_items_of_a_set_stay_one_release() { let poster = "Poster49118 "; let a = release_identity(poster, r#"ai-engineer-complete [401/465] - "003 Defining Tools with JSON Schema.mp4" yEnc"#, "003 Defining Tools with JSON Schema.mp4"); let b = release_identity(poster, r#"ai-engineer-complete [422/465] - "006 ai-lesson-4-5-web-search-tools.zip" yEnc"#, "006 ai-lesson-4-5-web-search-tools.zip"); assert_eq!(a.key, b.key); assert_eq!(a.name, "ai engineer complete"); let sc = |t: &str| release_identity("p", &format!("Spotlight - 83xx's [16/19] - \"{t}.zip\" yEnc"), &format!("{t}.zip")); assert_eq!(sc("SC8367-12 - Wynette, Tammy - Singing My Song").key, sc("SC8367-13 - Jones, George - He Stopped Loving Her Today").key); } } #[cfg(test)] mod poster_rule_tests { use super::{judge, JudgedFile, Rules, Verdict}; fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> { let groups = vec!["alt.binaries.erotica".to_string()]; let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "", poster, segments: 1, bytes: 200_000 }]; match judge("k", &files, true, rules) { Verdict::Keep => None, Verdict::Drop { rule, key, .. } => Some((rule, key)), } } #[test] fn a_dropped_posters_uploads_are_dropped() { let rules = Rules::defaults(Default::default()); assert_eq!(verdict("@_CelebMovie_@", &rules), Some(("poster", "celebmovie".to_string()))); assert_eq!(verdict("@__Celeb_Movie__@", &rules), Some(("poster", "celebmovie".to_string()))); assert_eq!(verdict("NudeCelebs@CelebWow.com", &rules), Some(("poster", "celebwow".to_string()))); assert_eq!(verdict("@nude-celebs", &rules), Some(("poster", "nudecelebs".to_string()))); assert_eq!(verdict("celeborn@not.here (Celeborn)", &rules), None); assert_eq!(verdict(r#""CelebTA.com" "#, &rules), None); let none = Rules { dropped_posters: vec![], ..Rules::defaults(Default::default()) }; assert_eq!(verdict("@_CelebMovie_@", &none), None); } } #[cfg(test)] mod single_archive_rule_tests { use super::{judge, JudgedFile, Rules, Verdict}; const NYUU: &str = "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"; fn verdict(name: &str, poster: &str, groups: usize, segments: u32, bytes: u64, settled: bool, rules: &Rules) -> Option<(&'static str, String)> { let groups: Vec = (0..groups).map(|i| format!("alt.binaries.g{i}")).collect(); let files = [JudgedFile { name, total_parts: segments, groups: &groups, first_message_id: NYUU, poster, segments, bytes }]; match judge("k", &files, settled, rules) { Verdict::Keep => None, Verdict::Drop { rule, key, .. } => Some((rule, key)), } } #[test] fn stub_archives_by_article_size_groups_or_webmail() { let rules = Rules::defaults(Default::default()); // Power-Post's 250 kB articles, two groups, 12 MB: the 2016 wave. assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com (Jahnunsinos007)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "AW".to_string()))); assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "Yenc@power-post.org (Yenc-PP-A&A)", 2, 48, 12_000_000, true, &rules), Some(("stub-archive", "A".to_string()))); // Five groups, 319 kB in two articles: the 2015 a.b.cracks wave (under 1 MB, so the groups leg alone). assert_eq!(verdict("Big Hero 6 (2014) 720p BDRip [COMPRESSED].rar", "Yenc@power-post.org (Yenc-PP-A&A)", 5, 2, 319_000, true, &rules), Some(("stub-archive", "X".to_string()))); // A webmail poster with 650 kB articles: the 2014 "leaked Sex Video" wave. assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 400, 260_000_000, true, &rules), None, "over 100 MB without the article-size leg"); assert_eq!(verdict("Leaked Sex Video.rar", "first.last88@gmail.com", 2, 100, 65_000_000, true, &rules), Some(("stub-archive", "W".to_string()))); // Settled only: a par2 can still join an open release. assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, false, &rules), None); } #[test] fn small_and_single_group_and_split_archives_are_kept() { let rules = Rules::defaults(Default::default()); // An e-book: one or two small articles always have small articles (LOG.md (126)). assert_eq!(verdict("Maeve Binchy - Tara Road (epub).rar", "kie@home.now (KiE)", 2, 2, 521_000, true, &rules), None); // One group: most real single rars. assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 1, 48, 12_000_000, true, &rules), None); // Modern 700 kB articles, no webmail, two groups. assert_eq!(verdict("Some.Movie.2014.720p.BluRay-GRP.rar", "poster@example.org", 2, 100, 70_000_000, true, &rules), None); // A volume, not a single archive; a hash-named 2 GB upload from a gmail-looking poster. assert_eq!(verdict("Some.Movie.2014.part01.rar", "a@gmail.com", 2, 48, 12_000_000, true, &rules), None); assert_eq!(verdict("cbcd32359460df1fa9b7d01d8720.rar", "b648de0d1c0f7a9a77be273d@gmail.com", 2, 3163, 2_340_000_000, true, &rules), None); // ROT13-named volumes of passworded videos in a.b.erotica, 258 kB articles, two groups, 1-270 MB; and the // 100 MB cap on every leg (LOG.md (128)). assert_eq!(verdict("Ghful.17.12.07.Qnvvfl.Fgbar.cneg3.rar", "nospam@nospam.nl (EvH)", 2, 5, 1_147_000, true, &rules), None); assert_eq!(verdict("Ghful.16.07.05.Wbwb.Xvff.cneg1.rar", "nospam@nospam.nl (EvH)", 2, 1051, 271_000_000, true, &rules), None); assert_eq!(verdict("Tinasoft EasyCafe 2.2.14 by OmerSalaj.rar", "Seven7i@mail.com ({Seven7i})", 4, 463, 117_000_000, true, &rules), None); assert_eq!(verdict("Tipard DVD Ripper - 10 0 12 x86 x64 incl Patch.rar", "Seven7i@mail.com ({Seven7i})", 4, 244, 62_000_000, true, &rules), Some(("stub-archive", "AXW".to_string()))); let off = Rules { drop_stub_archives: false, ..Rules::defaults(Default::default()) }; assert_eq!(verdict("Chat Watch 1.2.1.zip", "jahnunsinos007@gmail.com", 2, 48, 12_000_000, true, &off), None); // A karaoke zip crossposted to a.b.test with Power-Post's articles: its group exempts it (LOG.md (127)). let groups = vec!["alt.binaries.sounds.karaoke".to_string(), "alt.binaries.test".to_string()]; let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl.zip", total_parts: 13, groups: &groups, first_message_id: NYUU, poster: "home@home.com (TexasDj)", segments: 13, bytes: 3_300_000 }]; assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep)); } #[test] fn nonsense_pieces() { let rules = Rules::defaults(Default::default()); // One 790 kB article under a release-like name: the newznzb.* pieces. assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string()))); // A 3 kB cue/log sidecar rar with a year in its name: under this rule's floor (the tiny-archive rule's outside the music groups). assert_eq!(verdict("The Smiths-the queen is dead(Remaster)(2011).rar", "Yenc@power-post.org (Sir Cedrik)", 1, 1, 3_000, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); // A book (a year or a format in its name, written with spaces), a dotted one too: not a piece. assert_eq!(verdict("Kevin Ashman - [Roman 02] - The Rise of Caratacus (2020) (epub).rar", "fake@address.com (Psychopath)", 2, 1, 303_000, true, &rules), None); assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 382_000, true, &rules), None); assert_eq!(verdict("Vieweg.Teubner.Verlag.-.Numerische.Mathematik.2011.German.RETAIL.eBook-BitBook.rar", "saggir@domain.com", 1, 1, 113_000, true, &rules), None); // Two articles, or no release-like name, or scrambled: not this rule. assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 2, 790_000, true, &rules), None); assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None); assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 790_000, true, &rules), None); let off = Rules { drop_nonsense_pieces: false, ..Rules::defaults(Default::default()) }; assert_eq!(verdict("Some.Show.S01E01.1080p.WEB-DL.rar", "poster@example.org", 1, 1, 790_000, true, &off), None); } #[test] fn tiny_archives() { let rules = Rules::defaults(Default::default()); // One article of 2-100 kB under a release name, dotted or with spaces: a RAR holding ".exe" (LOG.md (131)). assert_eq!(verdict("Humans.S01E04.German.DD20.Dubbed.DL.1080p.BD.x264-TVS.rar", "Yanc@power-post.org <Yanc@power-post.org>", 1, 1, 15_084, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); assert_eq!(verdict("Cicero Magazin fuer politische Kultur Mai 05 2016.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 17_478, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); // A scene name without a year, a tag or a group suffix: not release-like, so kept (the flood's porn names mostly carry one). assert_eq!(verdict("[BigTitsAtWork Brazzers] Eva Angelina (Camera Cums In Handy).rar", "Yanc@power-post.org", 2, 1, 23_449, true, &rules), None); assert_eq!(verdict("Graphisoft.Archicad.v14.0.GERMAN.iSO.X86-PillePalle.rar", "WarezGeek@WarezGeek.org (WarezGeek)", 1, 1, 7_869, true, &rules), Some(("tiny-archive", "nyuu".to_string()))); // Subtitles, a crack, a trainer, a cover, an e-book: the real tiny archives. assert_eq!(verdict("The.Big.Bang.Theory.S04E13.720p.HDTV.subs.rar", "henk@example.org", 1, 1, 38_000, true, &rules), None); assert_eq!(verdict("Tipard.DVD.Ripper.v10.0.12.Crack-XXX.rar", "poster@example.org", 1, 1, 60_000, true, &rules), None); assert_eq!(verdict("Far.Cry.4.v1.7.0.Plus.12.Trainer-FLiNG.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None); assert_eq!(verdict("Some Band - Album (2011) covers.rar", "poster@example.org", 1, 1, 45_000, true, &rules), None); assert_eq!(verdict("One.Wrong.Turn.Deanna.Lynn.Sletten.epub.rar", "fake@address.com (Psychopath)", 1, 1, 38_000, true, &rules), None); // A karaoke zip in its group; a plain name; a scrambled one; two articles; under the floor; above 100 kB is the nonsense-piece rule's. let groups = vec!["alt.binaries.sounds.karaoke".to_string()]; let files = [JudgedFile { name: "KV-1055806 - Walker Hayes - U Gurl (2017).zip", total_parts: 1, groups: &groups, first_message_id: NYUU, poster: "home@home.com", segments: 1, bytes: 60_000 }]; assert!(matches!(judge("k", &files, true, &rules), Verdict::Keep)); assert_eq!(verdict("holiday snaps.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None); assert_eq!(verdict("nYDeXeF49dieZ1J7flK-JFzZ3j.rar", "poster@example.org", 1, 1, 20_000, true, &rules), None); assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 2, 20_000, true, &rules), None); assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 1_500, true, &rules), None); assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 150_000, true, &rules), Some(("nonsense-piece", "nyuu".to_string()))); assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, false, &rules), None); let off = Rules { drop_tiny_archives: false, ..Rules::defaults(Default::default()) }; assert_eq!(verdict("Humans.S01E04.German.DL.1080p.BD.x264-TVS.rar", "poster@example.org", 1, 1, 20_000, true, &off), None); } } #[cfg(test)] mod image_set_tests { use super::{release_identity, sized_name, subject_remainder, subject_remainder_re}; #[test] fn random_posters_are_no_part_of_the_key() { use super::random_poster; let ftd = "-Alight_Upon_the_Lake_(Live_at_the_Jazz_Showcase_Jun_1975)-WEB-2026-FTD"; let a = release_identity("N9vplIaavBCz@aCTVoAsk.JSE", &format!("[1/6] - \"Yusef_Lateef{ftd}.par2\" yEnc 1744"), &format!("Yusef_Lateef{ftd}.par2")); let b = release_identity("FVWWHku0fGAg@QfaTz0g0.RVH", &format!("[3/6] - \"Yusef_Lateef{ftd}.part2.rar\" yEnc 104857600"), &format!("Yusef_Lateef{ftd}.part2.rar")); assert_eq!(a.key, b.key); assert!(random_poster("N9vplIaavBCz@aCTVoAsk.JSE")); assert!(!random_poster("Captain Salvo <Capt@Salvo.com>")); assert!(!random_poster("Poster49118 <Poster49118@generic.mail>")); assert!(!random_poster("w.merklein@freenet.de (muffin)")); } #[test] fn image_sets_without_suffixes() { use super::image_set; for (stem, want) in [ ("stp250x001", "stp250x"), ("cutie-071-032", "cutie-071-"), ("katelg010_10", "katelg"), ("kate112lg035_35", "kate112lg"), ("!!hgrscans1097_2009", "!!hgrscans"), ("pj02kristina_24of40_1600", "pj02kristina_"), ("keira_s04_l keira_s04_0040_l", "keira_s04_l keira_s04_"), ("IMG_2676", "IMG_"), ("2877-041", "2877-"), ("051 - White leather kate054001_1_1", "051 - White leather kate"), ("108 - Schoolwall kate108lg029_28", "108 - Schoolwall kate108lg"), ("kurozo-2006-7-26-15-21", ""), ("bwp2_037_5", "bwp2_"), ("Set 008 10of63 new0464", "Set 008 "), ("202/430 princess-079-058", ""), ("( 098 /500) 002 148", ""), ("New Folder (17) sexypattycake_0903040445[1]", "New Folder ("), ("092_081_2000x1500", "092_"), ("KP0027_197_00", "KP"), ("01079-106-2048px", "01079-"), ("diana-050-052[1][1]", "diana-050-"), ("sss-70556-3474182-23-10825418-pc", ""), ("Alena p 26663_alena-p-2-8323_123_39lo", ""), ("she has sensual legs 172 _ 172", "she has sensual legs "), ("jamie.lee.curtis_the.tailor.of.panama_17_123_244lo", "jamie.lee.curtis_the.tailor.of.panama_"), ("JimAndJoey Kindly Husband-to-Be", ""), ] { assert_eq!(image_set(stem), want, "{stem}"); } } #[test] fn image_sets_by_their_numbered_name() { let silver = |n: &str| release_identity("p", &format!(r#"The excellent STIRILING SILVER - Always popular - "{n}" yEnc 320794 bytes"#), n); assert_eq!(silver("stp250x001.jpg").key, silver("stp250x002.jpg").key); assert_ne!(silver("stp250x001.jpg").key, silver("stp305x001.jpg").key); let mint = |n: &str| release_identity("Cert", &format!(r#"I'm not a breath mint, but I am not all there either. [{n}" yEnc]"#), n); assert_eq!(mint("JimAndJoey Kindly Husband-to-Be.JPG").key, mint("Oops OnlyOneMore.JPG").key); } #[test] fn a_folders_images_posted_with_their_sizes_are_one_release() { let celeb = |n: &str, size: u32| release_identity("@_CelebMovie_@", &format!(">>>>> Jamie Lee Curtis - Nude Celebrity Pictures <<<<< - {n} {size} bytes"), n); let keys: std::collections::HashSet<String> = ["!index0001.jpg", "00-00-02_123_842lo.jpg", "00_00_13.jpg", "bboobs02-curtis-jl.jpg", "0c4_jamie_lee_curtis_15.jpg"].iter().enumerate().map(|(i, n)| celeb(n, 75220 + i as u32).key).collect(); assert_eq!(keys.len(), 1); assert_eq!(celeb("00_00_13.jpg", 58748).source, "subject"); assert_ne!(celeb("00_00_13.jpg", 1).key, release_identity("@_CelebMovie_@", ">>>>> Carmen Electra - Xmas Nudes <<<<< - 00_00_13.jpg 1 bytes", "00_00_13.jpg").key); let ctp = |n: &str| release_identity("new@update.org (New CTP update)", &format!("*** NEW *** CTP update & adds - {n} - 18441 bytes"), n); assert_eq!(ctp("cc_CG2008_v10_01_Stephanie_Emma.jpg").key, ctp("cc_CastingCall_LasVegas_01_Kristin_00_Header1.jpg").key); assert!(sized_name("x - a.jpg 1,234 Bytes", "a.jpg")); assert!(!sized_name(r#"x - "a.jpg" yEnc 320794 bytes"#, "a.jpg")); assert!(!sized_name("x - a.jpg 5 megabytes", "a.jpg")); assert!(!sized_name("x - a.jpg yEnc", "a.jpg")); } #[test] fn other_quoted_text_and_text_separators_name_the_release() { let cc = |name: &str, n: &str| release_identity("p", &format!(r#">>> CREATIVE COMMONS NZB <<< "{name} " - File 1 of 9: "creative_commons_nzb_{n}.rar" yEnc"#), &format!("creative_commons_nzb_{n}.rar")); assert_ne!(cc("0190 alarm 5.0", "0190_alarm_5.0_").key, cc("portable irfanview 4.22", "portable_irfanview_4.22").key); let vlad = |dir: &str, total: u32, n: &str| release_identity("p", &format!(r#"Posting & deleting, read 0 file "E:\VladModels\y038_Irina\{dir}" Day 5/6 [01/{total}] - {n} yEnc"#), n); assert_eq!(vlad("185", 77, "P8250001.JPG").source, "subject"); assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key); } #[test] fn hex_ids_dated_stems_and_season_packs() { let nz = |id: &str, i: u32, n: &str| release_identity("nEwZ", &format!(r#"[PRiVATE] Rosewood.S02E08.SPANiSH.1080p.HDTV.x264-FCC [{id}] [newzNZB] [{i:02}/22] - "{n}" yEnc"#), n); assert_eq!(nz("1f1793f68d5321", 1, "[05cbdb538a]_x.nfo").key, nz("65e6464bc03cad", 2, "[e8a4e2c182]_x.r03").key); let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Coast To Coast Radio Show - "{n}" yEnc"#), n); assert_eq!(show("09-30-2008_CTC.nfo").key, show("09-30-2008_CTC.r00").key); assert_ne!(show("09-30-2008_CTC.r00").key, show("10-01-2008_CTC.r00").key); let ep = |i: u32, n: &str| release_identity("p", &format!(r#"[{i:02}/36] "{n}""#), n); assert_eq!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").key, ep(2, "Pokemon - S02E02 - Gotta Catch Ya Later.mkv").key); assert_eq!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").name, "Pokemon S02"); assert_ne!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").key, ep(1, "Pokemon - S03E01 - x.mkv").key); } #[test] fn scene_index_files_go_with_the_release() { let p = "SavageCobra <x@y.nhc>"; let a = release_identity(p, "00-laura_van_dam_-_capture_radio_113-cable-10-10-2026-talion.m3u", "00-laura_van_dam_-_capture_radio_113-cable-10-10-2026-talion.m3u"); let b = release_identity(p, "Laura_van_Dam_-_Capture_Radio_113-CABLE-10-10-2026-TALiON.vol000+001.par2", "Laura_van_Dam_-_Capture_Radio_113-CABLE-10-10-2026-TALiON.vol000+001.par2"); assert_eq!(a.key, b.key); assert_eq!(b.name, "Laura_van_Dam_-_Capture_Radio_113-CABLE-10-10-2026-TALiON"); } #[test] fn split_rar_volumes_and_sizes_after_the_counter() { assert_eq!(super::file_stem("rld-gtre.r13.002"), "rld-gtre"); for (n, stem) in [ ("BD_NoTHing018.s41", "BD_NoTHing018"), ("x8ZRmX5N83mrokOwqJtoIGHTrqXQrgCyj.s00", "x8ZRmX5N83mrokOwqJtoIGHTrqXQrgCyj"), ("the.office.us.s01.rar", "the.office.us.s01"), ("Some.Show.S01", "Some.Show.S01"), ("Electric.Rain.Swift.3D.PS.v1.0.142.x64", "Electric.Rain.Swift.3D.PS.v1.0.142.x64"), ("App.x86", "App.x86"), ("Licence 700790900887990.z18", "Licence 700790900887990"), ("Licence 077887994400668.x12", "Licence 077887994400668"), ("Reply 987986177631730.x004", "Reply 987986177631730"), ("goh1Phoz5ahXu.par.vol006+08.par2", "goh1Phoz5ahXu"), ("brSWc0Q2KWblQpogOmgbn9.part.par2", "brSWc0Q2KWblQpogOmgbn9"), ("x.parts.par2", "x.parts"), ("Some.Movie.2010.x264", "Some.Movie.2010.x264"), ("chappelle1.part02.rev", "chappelle1"), ("chappelle1.rev", "chappelle1"), ("VwE3i90MVX9dqMFuDfzdS7Q6.part1.par2", "VwE3i90MVX9dqMFuDfzdS7Q6"), ("VwE3i90MVX9dqMFuDfzdS7Q6.part1.vol00+34.par2", "VwE3i90MVX9dqMFuDfzdS7Q6"), ("Movie.Part1.mkv", "Movie.Part1"), ("NZPECQUztWOponTpCratvvTPDBXBjvHIULOUPvSH.180.bc", "NZPECQUztWOponTpCratvvTPDBXBjvHIULOUPvSH"), ("x.12345.bc", "x.12345"), ] { assert_eq!(super::file_stem(n), stem, "{n}"); assert_eq!(super::file_stem_re(n), stem, "{n}"); } assert_eq!(super::size_after_name("(Hot MILF getting banged hard)[27/70] - 2407.jpg - 28", "2407.jpg"), "(Hot MILF getting banged hard)[27/70] - 2407.jpg"); assert_eq!(super::size_after_name(r#"Show [01/10] - "a.jpg" - 28"#, "a.jpg"), r#"Show [01/10] - "a.jpg" - 28"#); assert_eq!(super::size_after_name("Show - Episode 05", "x.mkv"), "Show - Episode 05"); for (n, cut) in [ ("lois 1258 Billy Joel - Extremes.part01.rar lois 1258 Billy Joel - Extremes ", "lois 1258 Billy Joel - Extremes.part01.rar"), ("GrooveCoverage.part26.rarGrooveCoveragebestof", "GrooveCoverage.part26.rar"), ("Goga Sekulic.part134.rar - Goga Sekulic - Diskografija (2000-2014).", "Goga Sekulic.part134.rar"), ("HASf293jLAIWUELvhaG3.part091.rar HASf293jLAIWUELvhaG3", "HASf293jLAIWUELvhaG3.part091.rar"), ("Movie.rar.Collection.mkv", "Movie.rar.Collection.mkv"), ("Angry.Birds.Rio.v1.2.2.cracked.READ.NFO-THETA.rar", "Angry.Birds.Rio.v1.2.2.cracked.READ.NFO-THETA.rar"), ("x.part01.rar", "x.part01.rar"), ] { assert_eq!(super::run_on(n), cut, "{n}"); } let lois = |i: &str, n: &str| super::release_identity("p", &format!(r#"(thnx to original poster) [{i}/92] - "{n}" yEnc"#), n); assert_eq!( lois("02", "lois 1258 Billy Joel - Extremes.part01.rar lois 1258 Billy Joel - Extremes ").key, lois("01", "lois 1258 Billy Joel - Extremes.par2 lois 1258 Billy Joel - Extremes ").key ); let jb = |i: &str, n: &str| super::release_identity("JBinUp", &format!(r#"[{i}/58] - Summer Power Up - "{n}" yEnc"#), n); assert_eq!(jb("01", "www.realmom.info - XVID- Farbiges Krabbeltier.r00").key, jb("54", "www.realmom.info - XVID-Farbiges Krabbeltier.vol03+04.par2").key); let lop = |i: &str, total: &str, n: &str| super::release_identity("p", &format!(r#"[{i}/{total}] - "{n}" yEnc"#), n); assert_eq!(lop("001", "559", "lop1001.jpg").key, lop("002", "559", "lop1002.jpg").key); assert_ne!(lop("001", "559", "lop1001.jpg").key, lop("001", "1018", "lop5001.jpg").key); assert_ne!( lop("022", "457", "r86_022 - Gwen Guthrie - Ain't Nothing Going on but the Rent.mp3").key, lop("022", "333", "r98_333 - Jellie - Roller Coaster.mp3").key ); let town = |i: &str, n: &str| super::release_identity("Kim <Jong-Un@town.ag>", &format!(r#"[ TOWN ] [ www.town.ag ] [ TV ] [{i}/20] - "{n}" yEnc"#), n); assert_eq!(town("01", "Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar").key, town("18", "Z3mMTMPsoT4ikE_Di3fKnNF21.vol4+4.par2").key); assert_ne!(town("01", "Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar").key, town("01", "i3DpbaTUJT4L6V48fEBdbVV46.part01.rar").key); let show = |i: &str, n: &str| super::release_identity("p", &format!(r#"Some.Show.S01E01 [{i}/20] - "{n}" yEnc"#), n); assert_eq!(show("01", "The.Big.Bang.Theory.S01E01.part01.rar").key, show("20", "The.Big.Bang.Theory.S01E01.nfo").key); let tender = |i: &str, n: &str| super::release_identity("p", &format!("Tender TEEN SEX in HD - File {i} of 435 - {n}"), n); assert_eq!(tender("001", "a1 (2).jpg").key, tender("002", "a1.jpg").key); let rv = |i: &str, total: &str, n: &str| super::release_identity("p", &format!(r#"[{i}/{total}] - "{n}" yEnc"#), n); assert_eq!(rv("01", "69", "df2BkeRxIvHcWrsWwwz.part01.cer").key, rv("69", "69", "df2BkeRxIvHcWrsWwwz.part69.bdmv").key); assert_eq!(rv("01", "32", "jA0eSSUMat8tQtIKUyPMA.part01").key, rv("32", "32", "jA0eSSUMat8tQtIKUyPMA.part32").key); assert_ne!(rv("01", "2", "Some.Movie.Part01.mkv").key, rv("02", "2", "Some.Movie.Part02.mkv").key); let dump = |i: &str, n: &str| super::release_identity("janusch", &format!(r#"(????) [{i}/15816] - "{n}""#), n); assert_ne!(dump("00005", "K-ML 3.8.233").key, dump("00006", "K-ML 3.8.239").key); let mdg = |post: &str, i: &str, n: &str| super::release_identity("p", &format!(r#"{post} - ({i}/90) "{n}" - 16,14 MB - yEnc"#), n); assert_eq!(mdg("MDG005b", "01", "IMG_2307.JPG").key, mdg("MDG005b", "02", "IMG_2308.jpg").key); assert_ne!(mdg("MDG005b", "01", "IMG_2307.JPG").key, mdg("MDG006a", "01", "IMG_4410.JPG").key); // The whole text before "(1/1)" is the name; the size follows the segment counter. let alexi = |i: &str, size: &str| { let n = format!("{i}of37 ## Alexi uses strange toys to make herself come ## alexis0{i}.jpg"); super::release_identity("p", &format!("{n} {size}"), &n) }; assert_eq!(alexi("01", "57295b").key, alexi("02", "32202b").key); let milf = |i: &str, n: &str, kb: &str| super::release_identity("p", &format!("(Hot MILF getting banged hard)[{i}/70] - {n} - {kb}"), n); assert_eq!(milf("27", "2407.jpg", "28").key, milf("28", "2408.jpg", "31").key); assert_eq!(super::file_stem_re("rld-gtre.r13.002"), "rld-gtre"); let k = |i: u32, size: u32, n: &str| release_identity("Kingdn", &format!(r#"Buffy The Vampire Slayer s02e04 [{i}/32] - [{size}] "{n}" yEnc"#), n); assert_eq!(k(26, 4929, "Buffy s02e04 xvid-tns.part24.rar").key, k(27, 400, "Buffy s02e04 xvid-tns.vol00+01.PAR2").key); } #[test] fn totals_padded_wider_than_the_index_are_no_counters() { let who = |i: u32, n: &str| release_identity("p", &format!(r#"Very Best Of The Who [{i:02}/{:03}] - "{n}" yEnc"#, i * 10), n); assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, "The Who - 02 - Anyway.mp3").key); assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").file_total, None); assert_eq!(super::counter_total(r#"x [001/010] - "a.rar""#).as_deref(), Some("10")); } #[test] fn dates_lead_no_numbered_items_and_repeated_names_go() { let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n); assert_eq!(show("09-30-2008_BAK_JRS.nfo").key, show("09-30-2008_BAK_JRS.r00").key); assert_ne!(show("09-30-2008_BAK_JRS.r00").key, show("10-01-2008_BAK_JRS.r00").key); let nelet = |n: &str, i: u32| release_identity("p", &format!(r#"({n}) [{i:02}/58] - "{n}" yEnc"#), n); assert_eq!(nelet("NELET11.par2", 1).key, nelet("NELET11.part01.rar", 2).key); let wl = |n: &str| release_identity("p", &format!(r#"(01/26) "{n}" "Silicon Dream - Time Machine_BY_WaRLoRD```` yEnc"#), n); assert_eq!(wl("01 Time Machine.mp3").key, wl("02 Jimmy Dean Loved Marilyn.mp3").key); } #[test] fn host_and_sponsor_without_counter() { let tn = |tag: &str, n: &str| release_identity("p", &format!(r#"(www.Thunder-News.org) >{tag}< <Sponsored by Secretusenet> - "{n}" yEnc"#), n); assert_eq!(tn("XXX", "ps-itb10b.r00").key, tn("XXX", "ps-itb10b.par2").key); assert_ne!(tn("XXX", "ps-itb10b.r00").key, tn("XXX", "swe6-legsa.r00").key); assert_eq!(tn("XXX", "ps-itb10b.r00").source, "stem"); let named = tn("For.Rent.XXX.720p.Bluray.x264-DPXXXHD", "dp-for.rent.720p.x264.r00"); assert_eq!(named.source, "subject"); assert_eq!(named.key, tn("For.Rent.XXX.720p.Bluray.x264-DPXXXHD", "dp-for.rent.720p.x264.par2").key); assert_eq!(named.name, ">For Rent XXX 720p Bluray x264 DPXXXHD<"); let counted = release_identity("p", r#"(www.Thunder-News.org) >XXX< [01/20] - "ps-itb10b.r00" yEnc"#, "ps-itb10b.r00"); assert_eq!(counted.source, "subject"); } #[test] fn hashed_stems_volume_numbers_and_scene_tracks() { let u4e = |n: &str| release_identity("p", &format!(r#"Seit 1753 Tagen - usenet4ever.info - "{n}" yEnc"#), n); assert_eq!(u4e("055631a1_b6c820bc_a856086c_85112a5-u4e.part01.rar").key, u4e("055631a1_b6c820bc_a856086c_85112a5-u4e.par2").key); assert_ne!(u4e("055631a1_b6c820bc_a856086c_85112a5-u4e.part01.rar").key, u4e("4cc7c4c6_d3c2fecf_0bff51d6_d10df86-u4e.part01.rar").key); let gb = |n: &str| release_identity("p", &format!(r#"German Board - [Usenet4ever.info] und [SecretUsenet.com] - "{n}" yEnc"#), n); assert_eq!(gb("6c57319fbae505679737bd46a76193a1-u4e.part6.rar").key, gb("6c57319fbae505679737bd46a76193a1-u4e.par2").key); assert_ne!(gb("6c57319fbae505679737bd46a76193a1-u4e.part6.rar").key, gb("6fc0ff192a355703b3e81557893013b1-u4e.part6.rar").key); assert_ne!(gb("21.Jump.Street.2012.BDRip.AC3.German.XviD-POE.part01.rar").key, gb("Ice.Age.4.Voll.verschoben.2012.BDRip.AC3.German.XviD-POE.part01.rar").key); assert_eq!(gb("21.Jump.Street.2012.BDRip.AC3.German.XviD-POE.part01.rar").key, gb("21.Jump.Street.2012.BDRip.AC3.German.XviD-POE.par2").key); let mac = |i: &str, n: &str| release_identity("p", &format!(r#"[1598]-[FULL]-[#a.b.teevee@EFNet]-[ MacGyver.S01E03.DVDRip.XviD-MEDiEVAL ]-[{i}/38] - "{n}" yEnc"#), n); assert_eq!(mac("04", "sample.macgy.s01e03-med.avi").key, mac("05", "macgy.s01e03-med.r00").key); let pck = |i: &str, n: &str| release_identity("p", &format!(r#"(????) [{i}/2195] - "{n}" yEnc"#), n); assert_eq!(pck("0659", "DN.VPL PCK.1508").key, pck("0660", "DN.VPL PCK.1509").key); assert_eq!(pck("0659", "DN.VPL PCK.1508").name, "DN.VPL PCK.1508"); let movie = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/30] - "{n}" yEnc"#), n); assert_ne!(movie("01", "Movie.2008").key, movie("02", "Movie.2009").key); let track = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/14] - "{n}" yEnc"#), n); assert_eq!(track("06", "02-repiet_-_dont_think_twice_(extended_mix).mp3").key, track("07", "03-repiet_-_other_song.mp3").key); assert_eq!(track("06", "02-repiet_-_dont_think_twice_(extended_mix).mp3").name, "repiet"); assert_eq!(track("01", "01-2bm_mq-real_rap_(intro).mp3").name, "2bm_mq"); let meow = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/13] - "{n}" yEnc"#), n); let tracks = meow("05", "01-ammar-turk_uit_de_kroeg.mp3").key; for n in ["00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg", "00-ammar-turk_uit_de_kroeg-single-web-nl-2022.nfo", "00-ammar-turk_uit_de_kroeg-single-web-nl-2022.sfv"] { assert_eq!(meow("01", n).key, tracks, "{n}"); } assert_ne!(meow("01", "00-frank_van_etten-wie_heeft_jouw_lach_gestolen-single-web-nl-2022-cover.jpg").key, tracks); let blaise = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/14] - "{n}" yEnc"#), n); assert_eq!(blaise("07", "blaise_mc-in_hellstill_3-ep-web-2022-known.par2").key, blaise("03", "00-blaise_mc-in_hellstill_3-ep-web-2022.nfo").key); assert_eq!(blaise("09", "blaise_mc-in_hellstill_3-ep-web-2022-known.vol000+01.par2").key, blaise("05", "01-blaise_mc-intro.mp3").key); assert_eq!(blaise("01", "Some.Movie.2010.1080p.BluRay.x264-GRP.par2").name, "Some.Movie.2010.1080p.BluRay.x264-GRP"); assert_ne!( blaise("01", "Ace_Of_Base-Wheel_Of_Fortune-(861_545-2)-CDM-FLAC-1992-REETKEVER.par2").key, blaise("01", "Ace_Of_Base-Other-(861_546-2)-CDM-FLAC-1993-REETKEVER.par2").key ); } #[test] fn images_without_words_by_their_set() { let id = |n: &str, c: &str| release_identity("p", &format!(r#"##[ "{n}" ({c})"#), n); assert_eq!(id("diana-040-001.jpg", "01/68").key, id("diana-040-002.jpg", "02/68").key); assert_ne!(id("diana-040-001.jpg", "01/68").key, id("diana-407-081.jpg", "81/82").key); } #[test] fn unquoted_names_are_no_counters() { let shy = |n: &str| release_identity("p", &format!("{n} hardcore SHYLA STYLEZ & SHY LOVE"), n); assert_eq!(shy("shyla-shy-09-0195.jpg").key, shy("shyla-shy-09-0766.jpg").key); assert_eq!(shy("shyla-shy-09-0195.jpg").file_total, None); } #[test] fn dates_are_no_counters() { use super::counter_total; assert_eq!(counter_total("Avira AntiVir Rescue System 08-29-2011.rar"), None); assert_eq!(counter_total("Android Apps and Games Pack 27-02-2011.rar"), None); assert_eq!(counter_total("Supernova-clubbin (slamfm)-cable-18-08-2012-1king.rar"), None); assert_eq!(counter_total(r#"Some.Album [03/12] - "x.mp3""#).as_deref(), Some("12")); assert_eq!(counter_total(r#"File 07390 of 19411 "x.epub""#).as_deref(), Some("19411")); } #[test] fn more_splits_from_the_reindex() { use super::random_poster; let makz = "Makz@BlueRadiance.uni (makz)"; // The name as unquoted_name gives it (spaces kept), the subject collapsed and without its counter. let a = release_identity(makz, r"G:\posting\thegoodnudes\lara_bath 15of19 larabath_2.5.jpg", r"G:\posting\thegoodnudes\lara_bath 15of19 larabath_2.5.jpg"); let b = release_identity(makz, r"G:\posting\thegoodnudes\lara_bath 16of19 larabath_2.6.jpg", r"G:\posting\thegoodnudes\lara_bath 16of19 larabath_2.6.jpg"); assert_eq!(a.key, b.key); let mara = "Mara <guest@cox.net>"; let a = release_identity(mara, r#"RP - File 268 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3""#, "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3"); let b = release_identity(mara, r#"RP - File 269 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3""#, "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3"); assert_eq!(a.key, b.key); let mb = "misterb@sure.com (MisterB)"; let a = release_identity(mb, r#"Herb Ernst@256 - NMR -[07/17] "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3" - 8123 yEnc"#, "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3"); let b = release_identity(mb, r#"Herb Ernst@256 - NMR -[08/17] "Herb Ernst - Dreamflight II - 06 - Other.mp3" - 9001 yEnc"#, "Herb Ernst - Dreamflight II - 06 - Other.mp3"); assert_eq!(a.key, b.key); let destiny = |set: u32, i: u32| { let name = format!("destiny-{set:03}-{i:03}.jpg"); release_identity("p", &format!(r#"Newstar Destiny I set {set} [{i} of 60] "{name}" yEnc"#), &name) }; assert_eq!(destiny(1, 1).key, destiny(1, 2).key); assert_ne!(destiny(1, 1).key, destiny(63, 34).key); assert_eq!(destiny(63, 34).name, "Newstar Destiny I set 63"); for s in [r#"x - "a.mp3" - 8123 yEnc"#, "a8123 yEnc", "-8123 yEnc ", "yEnc", r#"set 1 [1 of 60] "x.jpg" yEnc"#, "Album 12 yEnc", "Album 1234 yEnc"] { assert_eq!(super::subject_remainder(s), super::subject_remainder_re(s), "{s}"); } assert!(random_poster("SOFZRw1-OTEd@EEFUbh.srG")); assert!(random_poster("vMgGYlZB9ZcTNk8WxYFGV_OOW@x6YR.abc")); assert!(!random_poster("w.merklein@freenet.de (muffin)")); } #[test] fn sizes_in_brackets_and_one_word_sets() { for s in [r#"Aletta Ocean - "100628-035.jpg" yEnc [347K]"#, r#"Corie - "2877-041.jpg" yEnc [290K]"#, "x - !!!blob2010-102katie009.jpg 336573 byt"] { assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}"); } assert_eq!(subject_remainder(r#"Aletta Ocean - "100628-035.jpg" yEnc [347K]"#), "Aletta Ocean"); let a = release_identity("p", r#"Corie - "2877-041.jpg" yEnc [290K]"#, "2877-041.jpg"); let b = release_identity("p", r#"Corie - "2877-042.jpg" yEnc [301K]"#, "2877-042.jpg"); assert_eq!(a.key, b.key); assert_eq!(a.name, "Corie"); } } }