Daan MeijerandClaude Opus 5.5
590436d02a
7ad4b70 also keyed rar uploads by artist and total: "VA - Best Of Shamania
Pro 2K16-(SHB 006)-WEB-2016-ZzZz" became 140 files for 20, Ace_Of_Base 72
for 12. Those group by their full name; only a name without capitals or
spaces (Meow's "blaise_mc-in_hellstill_3-ep-web-2022-known.par2") joins
the tracks.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1685 lines
82 KiB
Rust
1685 lines
82 KiB
Rust
//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools.
|
||
//! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u`
|
||
//! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS
|
||
//! `\s` (which includes Unicode spaces) is spelled out as SP.
|
||
|
||
use crate::js::{self, len16, sha1_hex};
|
||
use md5::{Digest, Md5};
|
||
use regex::bytes::Regex;
|
||
use std::sync::LazyLock;
|
||
|
||
/// JS `\s` as a byte pattern.
|
||
const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)";
|
||
|
||
fn re(pattern: &str) -> Regex {
|
||
Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap()
|
||
}
|
||
|
||
/// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all).
|
||
fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String {
|
||
let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) };
|
||
// The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8.
|
||
String::from_utf8(out.into_owned()).unwrap()
|
||
}
|
||
|
||
fn test(re: &Regex, s: &str) -> bool {
|
||
re.is_match(s.as_bytes())
|
||
}
|
||
|
||
static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||
vec![
|
||
// A part's par2 files ("X.part1.par2", "X.part1.vol00+34.par2").
|
||
(re(r"(?i)\.part[0-9]+(?:\.vol[0-9]+[+-][0-9]+)?\.par2$"), ""),
|
||
(re(r"(?i)\.(?:part|par)(?:\.vol[0-9]+[+-][0-9]+)?\.par2$"), ""),
|
||
(re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""),
|
||
(re(r"(?i)\.par2$"), ""),
|
||
(re(r"(?i)\.part[0-9]+\.(?:rar|rev)$"), ""),
|
||
(re(r"(?i)\.r[0-9]{2,3}$"), ""),
|
||
(re(r"(?i)\.(?:rar|rev)$"), ""),
|
||
(re(r"(?i)(?:\.[0-9]{1,4})?\.bc$"), ""),
|
||
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob|r[0-9]{2,3}))\.[0-9]{2,3}$"), "$1"),
|
||
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||
(re(r"(?i)\.tar$"), ""),
|
||
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||
]
|
||
});
|
||
|
||
/// fileStem with the regexes, as in the TypeScript (the reference for file_stem's tests).
|
||
#[allow(dead_code)]
|
||
pub fn file_stem_re(name: &str) -> String {
|
||
let trimmed = js::trim(name);
|
||
let mut stem = if old_volume_at(trimmed.as_bytes()) { replace(&OLD_RAR_VOLUME, trimmed, "", false) } else { trimmed.to_string() };
|
||
let mut changed = true;
|
||
while changed {
|
||
changed = false;
|
||
for (suffix, with) in SUFFIXES.iter() {
|
||
if !suffix.is_match(stem.as_bytes()) {
|
||
continue;
|
||
}
|
||
let next = replace(suffix, &stem, with, false);
|
||
if next != stem && !next.is_empty() {
|
||
stem = next;
|
||
changed = true;
|
||
}
|
||
}
|
||
}
|
||
stem
|
||
}
|
||
|
||
/// Old-style rar volumes after .r99 and split zip volumes, as the file's own extension, lower case; .xNN but not
|
||
/// .x32/.x64/.x86 (no lookahead here: the builds are put back by old_volume_at).
|
||
static OLD_RAR_VOLUME: LazyLock<Regex> = LazyLock::new(|| re(r"\.(?:[s-v][0-9]{2}|[xz][0-9]{2,3})$"));
|
||
|
||
/// Whether a name ends with an old rar or split zip volume (OLD_RAR_VOLUME without the .x builds).
|
||
fn old_volume_at(b: &[u8]) -> bool {
|
||
let len = b.len();
|
||
let digits = b.iter().rev().take_while(|c| c.is_ascii_digit()).count();
|
||
if !(2..=3).contains(&digits) || len < digits + 2 || b[len - digits - 2] != b'.' {
|
||
return false;
|
||
}
|
||
let letter = b[len - digits - 1];
|
||
let number = &b[len - digits..];
|
||
match letter {
|
||
b's'..=b'v' => digits == 2,
|
||
b'z' => true,
|
||
// Not the builds .x32, .x64, .x86, .x264, .x265.
|
||
b'x' => !matches!(number, b"32" | b"64" | b"86" | b"264" | b"265"),
|
||
_ => false,
|
||
}
|
||
}
|
||
static QUOTED: LazyLock<Regex> = LazyLock::new(|| re(r#""[^"]*""#));
|
||
static COUNTERS: LazyLock<Regex> = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)"));
|
||
static YENC_END: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$"));
|
||
static YENC: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b"));
|
||
static SEPARATORS: LazyLock<Regex> = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+"));
|
||
|
||
/// subjectRemainder with the regexes, as in the TypeScript (the reference for subject_remainder's tests).
|
||
#[allow(dead_code)]
|
||
pub fn subject_remainder_re(subject: &str) -> String {
|
||
let s = replace("ED, subject, " ", true);
|
||
let s = replace(&COUNTERS, &s, " ", true);
|
||
let s = remove_counters_and_sizes(&s);
|
||
let s = replace(&YENC_END, &s, " ", false);
|
||
let s = replace(&YENC, &s, " ", true);
|
||
let s = replace(&SEPARATORS, &s, " ", true);
|
||
finish_remainder(&s)
|
||
}
|
||
|
||
fn eq_ci(a: &[u8], b: &[u8]) -> bool {
|
||
a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_ascii_lowercase() == *y)
|
||
}
|
||
|
||
/// Ends in `suffix` (lowercase), ASCII case-insensitively.
|
||
fn ends_ci(s: &[u8], suffix: &[u8]) -> bool {
|
||
s.len() >= suffix.len() && eq_ci(&s[s.len() - suffix.len()..], suffix)
|
||
}
|
||
|
||
/// ASCII digits ending at `end`.
|
||
fn digits_back(s: &[u8], end: usize) -> usize {
|
||
s[..end].iter().rev().take_while(|c| c.is_ascii_digit()).count()
|
||
}
|
||
|
||
/// `.` + one of `exts` at the end: the length without it.
|
||
fn strip_ext(s: &[u8], exts: &[&[u8]]) -> Option<usize> {
|
||
for ext in exts {
|
||
let n = ext.len() + 1;
|
||
if s.len() >= n && s[s.len() - n] == b'.' && ends_ci(s, ext) {
|
||
return Some(s.len() - n);
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// `\.\d{3}$`: the length without it.
|
||
fn strip_three_digits(s: &[u8]) -> Option<usize> {
|
||
let n = s.len();
|
||
(n >= 4 && s[n - 4] == b'.' && s[n - 3..].iter().all(u8::is_ascii_digit)).then(|| n - 4)
|
||
}
|
||
|
||
/// \.\d{2,3}$: where a split volume's number starts (its dot).
|
||
fn strip_volume_digits(s: &[u8]) -> Option<usize> {
|
||
let n = s.len();
|
||
let d = s.iter().rev().take_while(|c| c.is_ascii_digit()).count();
|
||
((2..=3).contains(&d) && n > d && s[n - d - 1] == b'.').then(|| n - d - 1)
|
||
}
|
||
|
||
const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"];
|
||
const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"];
|
||
const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"];
|
||
const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"];
|
||
|
||
/// SUFFIXES[rule] once: the stem's new length when it matches.
|
||
fn strip_suffix_rule(s: &[u8], rule: usize) -> Option<usize> {
|
||
let n = s.len();
|
||
match rule {
|
||
// \.vol\d+[+-]\d+\.par2$
|
||
0 => {
|
||
if !ends_ci(s, b".par2") {
|
||
return None;
|
||
}
|
||
let d2 = digits_back(s, n - 5);
|
||
let sign = (n - 5).checked_sub(d2 + 1).filter(|_| d2 > 0)?;
|
||
if !matches!(s[sign], b'+' | b'-') {
|
||
return None;
|
||
}
|
||
let d1 = digits_back(s, sign);
|
||
let v = sign.checked_sub(d1 + 4).filter(|_| d1 > 0)?;
|
||
eq_ci(&s[v..v + 4], b".vol").then_some(v)
|
||
}
|
||
1 => ends_ci(s, b".par2").then(|| n - 5),
|
||
// \.part\d+\.(?:rar|rev)$
|
||
2 => {
|
||
if !ends_ci(s, b".rar") && !ends_ci(s, b".rev") {
|
||
return None;
|
||
}
|
||
let d = digits_back(s, n - 4);
|
||
let p = (n - 4).checked_sub(d + 5).filter(|_| d > 0)?;
|
||
eq_ci(&s[p..p + 5], b".part").then_some(p)
|
||
}
|
||
// \.r\d{2,3}$
|
||
3 => {
|
||
let d = digits_back(s, n);
|
||
let p = n.checked_sub(d + 2).filter(|_| (2..=3).contains(&d))?;
|
||
eq_ci(&s[p..p + 2], b".r").then_some(p)
|
||
}
|
||
4 => (ends_ci(s, b".rar") || ends_ci(s, b".rev")).then(|| n - 4),
|
||
// \.7z(\.\d{3})?$ and \.zip(\.\d{3})?$
|
||
5 | 6 => {
|
||
let ext: &[u8] = if rule == 5 { b".7z" } else { b".zip" };
|
||
if let Some(m) = strip_three_digits(s) {
|
||
if ends_ci(&s[..m], ext) {
|
||
return Some(m - ext.len());
|
||
}
|
||
}
|
||
ends_ci(s, ext).then(|| n - ext.len())
|
||
}
|
||
// (?<=\.(7z|…))\.\d{2,3}$: only the volume number goes.
|
||
7 => {
|
||
let m = strip_volume_digits(s)?;
|
||
// Also after a rar volume ("rld-gtre.r13.002"): \.r\d{2,3} before it.
|
||
strip_ext(&s[..m], &SPLIT_EXTS).or_else(|| strip_suffix_rule(&s[..m], 3)).map(|_| m)
|
||
}
|
||
8 => strip_ext(s, &SIDE_EXTS),
|
||
9 => strip_ext(s, &[b"tar"]),
|
||
10 => strip_ext(s, &PACK_EXTS),
|
||
11 => strip_ext(s, &MEDIA_EXTS),
|
||
// \.part\d+(?:\.vol\d+[+-]\d+)?\.par2$
|
||
12 => {
|
||
if !ends_ci(s, b".par2") {
|
||
return None;
|
||
}
|
||
let m = strip_suffix_rule(s, 0).unwrap_or(n - 5);
|
||
let d = digits_back(s, m);
|
||
let p = m.checked_sub(d + 5).filter(|_| d > 0)?;
|
||
eq_ci(&s[p..p + 5], b".part").then_some(p)
|
||
}
|
||
// \.(?:part|par)(?:\.vol\d+[+-]\d+)?\.par2$
|
||
14 => {
|
||
if !ends_ci(s, b".par2") {
|
||
return None;
|
||
}
|
||
let m = strip_suffix_rule(s, 0).unwrap_or(n - 5);
|
||
if m >= 5 && eq_ci(&s[m - 5..m], b".part") {
|
||
Some(m - 5)
|
||
} else if m >= 4 && eq_ci(&s[m - 4..m], b".par") {
|
||
Some(m - 4)
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
// (?:\.\d{1,4})?\.bc$
|
||
13 => {
|
||
if !ends_ci(s, b".bc") {
|
||
return None;
|
||
}
|
||
let m = n - 3;
|
||
let d = digits_back(s, m);
|
||
Some(if (1..=4).contains(&d) && m > d && s[m - d - 1] == b'.' { m - d - 1 } else { m })
|
||
}
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
/// fileStem: per-file suffixes stripped, repeatedly. The SUFFIXES regexes by hand: with them, a
|
||
/// file's description was most of the indexing time.
|
||
pub fn file_stem(name: &str) -> String {
|
||
let stem = js::trim(name);
|
||
let b = stem.as_bytes();
|
||
let mut len = b.len();
|
||
// Old-style rar volumes after .r99 (".s41"), as the file's own extension, lower case.
|
||
if old_volume_at(b) {
|
||
len -= b.iter().rev().take_while(|c| c.is_ascii_digit()).count() + 2;
|
||
}
|
||
let mut changed = true;
|
||
while changed {
|
||
changed = false;
|
||
// In the order of SUFFIXES: a part's par2 first, BCArchive volumes after the rar rules.
|
||
for rule in [12, 14, 0, 1, 2, 3, 4, 13, 5, 6, 7, 8, 9, 10, 11] {
|
||
if let Some(next) = strip_suffix_rule(&b[..len], rule) {
|
||
if next > 0 && next < len {
|
||
len = next;
|
||
changed = true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// The suffixes are ASCII: `len` is a character boundary.
|
||
stem[..len].to_string()
|
||
}
|
||
|
||
fn is_word(c: u8) -> bool {
|
||
c.is_ascii_alphanumeric() || c == b'_'
|
||
}
|
||
|
||
/// "yEnc" (any case) at `i` with a word boundary (ASCII, as in JS) on both sides.
|
||
fn yenc_at(b: &[u8], i: usize) -> bool {
|
||
i + 4 <= b.len() && eq_ci(&b[i..i + 4], b"yenc") && (i == 0 || !is_word(b[i - 1])) && (i + 4 == b.len() || !is_word(b[i + 4]))
|
||
}
|
||
|
||
/// `(\s+\d+)?\s*$`
|
||
fn yenc_tail(rest: &str) -> bool {
|
||
let after_spaces = rest.trim_start_matches(js::is_space);
|
||
if after_spaces.chars().all(js::is_space) {
|
||
return true;
|
||
}
|
||
if after_spaces.len() == rest.len() {
|
||
return false;
|
||
}
|
||
let after_digits = after_spaces.trim_start_matches(|c: char| c.is_ascii_digit());
|
||
after_digits.len() < after_spaces.len() && after_digits.chars().all(js::is_space)
|
||
}
|
||
|
||
/// subjectRemainder by hand: the same five replacements and trim.
|
||
pub fn subject_remainder(subject: &str) -> String {
|
||
let b = subject.as_bytes();
|
||
// "[^"]*" → ' '
|
||
let mut s1: Vec<u8> = Vec::with_capacity(b.len());
|
||
let mut i = 0;
|
||
while i < b.len() {
|
||
if b[i] == b'"' {
|
||
if let Some(j) = b[i + 1..].iter().position(|&c| c == b'"') {
|
||
s1.push(b' ');
|
||
i += j + 2;
|
||
continue;
|
||
}
|
||
}
|
||
s1.push(b[i]);
|
||
i += 1;
|
||
}
|
||
// \[\d+/\d+\] and \(\d+/\d+\) → ' '
|
||
let mut s2: Vec<u8> = Vec::with_capacity(s1.len());
|
||
let mut i = 0;
|
||
while i < s1.len() {
|
||
let close = match s1[i] {
|
||
b'[' => Some(b']'),
|
||
b'(' => Some(b')'),
|
||
_ => None,
|
||
};
|
||
if let Some(close) = close {
|
||
let a = s1[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let slash = i + 1 + a;
|
||
if a > 0 && s1.get(slash) == Some(&b'/') {
|
||
let d = s1[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if d > 0 && s1.get(slash + 1 + d) == Some(&close) {
|
||
s2.push(b' ');
|
||
i = slash + 2 + d;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
s2.push(s1[i]);
|
||
i += 1;
|
||
}
|
||
// Only ASCII was replaced, between whole characters.
|
||
let mut s3 = remove_counters_and_sizes(&String::from_utf8(s2).unwrap());
|
||
// \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits)
|
||
let b3 = s3.as_bytes();
|
||
if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) {
|
||
s3.truncate(at);
|
||
s3.push(' ');
|
||
}
|
||
// \byEnc\b → ' ', then runs of \s, -, –, ., :, | → ' '
|
||
let b3 = s3.as_bytes();
|
||
let mut s4 = String::with_capacity(s3.len());
|
||
let mut separator = false;
|
||
let mut i = 0;
|
||
while i < b3.len() {
|
||
if yenc_at(b3, i) {
|
||
// Its " " starts or joins a separator run.
|
||
if !separator {
|
||
s4.push(' ');
|
||
separator = true;
|
||
}
|
||
i += 4;
|
||
continue;
|
||
}
|
||
let c = s3[i..].chars().next().unwrap();
|
||
if js::is_space(c) || matches!(c, '-' | '.' | ':' | '|' | '\u{2013}') {
|
||
if !separator {
|
||
s4.push(' ');
|
||
separator = true;
|
||
}
|
||
} else {
|
||
s4.push(c);
|
||
separator = false;
|
||
}
|
||
i += c.len_utf8();
|
||
}
|
||
finish_remainder(&s4)
|
||
}
|
||
|
||
static OF_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b[\])]?"));
|
||
static DASH_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)-([0-9]+)\]"));
|
||
static SLASH_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\b([0-9]+)/([0-9]+)\b"));
|
||
static ZERO_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\b(0[0-9]*)-([0-9]+)\b"));
|
||
static SIZES: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b[0-9]+(?:[.,][0-9]+)?SP*(?:[kmgt]i?b|bytes?|byt)\b"));
|
||
static BRACKET_SIZES: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\[SP*[0-9]+(?:[.,][0-9]+)?SP*[kmgt]SP*\]"));
|
||
static EMPTY_BRACKETS: LazyLock<Regex> = LazyLock::new(|| re(r" ?(?:\[ ?\]|\( ?\)) ?"));
|
||
|
||
fn number(digits: &[u8]) -> f64 {
|
||
std::str::from_utf8(digits).unwrap().parse::<f64>().unwrap()
|
||
}
|
||
|
||
/// `s.replace(/…/g, counter)`: the match goes when its first number is at most its second.
|
||
fn replace_counters(re: &Regex, s: &str) -> String {
|
||
if !re.is_match(s.as_bytes()) {
|
||
return s.to_string();
|
||
}
|
||
let out = re.replace_all(s.as_bytes(), |c: ®ex::bytes::Captures| if number(&c[1]) <= number(&c[2]) { b" ".to_vec() } else { c[0].to_vec() });
|
||
String::from_utf8(out.into_owned()).unwrap()
|
||
}
|
||
|
||
/// The written-out counters, file sizes and empty brackets subjectRemainder removes after the [i/M] ones.
|
||
fn remove_counters_and_sizes(s: &str) -> String {
|
||
let s = replace_counters(&OF_COUNTER, s);
|
||
let s = replace_counters(&DASH_COUNTER, &s);
|
||
let s = replace_counters(&SLASH_COUNTER, &s);
|
||
let s = replace_counters(&ZERO_COUNTER, &s);
|
||
let s = replace(&SIZES, &s, " ", true);
|
||
let s = replace(&BRACKET_SIZES, &s, " ", true);
|
||
strip_number_before_yenc(&s)
|
||
}
|
||
|
||
/// `.replace(/\b\d+(\s+yEnc\s*)$/i, '$1')`: a size without a unit right before the closing yEnc.
|
||
fn strip_number_before_yenc(s: &str) -> String {
|
||
let tail_start = s.trim_end_matches(js::is_space).len();
|
||
let body = &s[..tail_start];
|
||
if body.len() < 4 || !body.is_char_boundary(body.len() - 4) || !body[body.len() - 4..].eq_ignore_ascii_case("yenc") {
|
||
return s.to_string();
|
||
}
|
||
let before = &body[..body.len() - 4];
|
||
let digits_end = before.trim_end_matches(js::is_space).len();
|
||
if digits_end == before.len() {
|
||
return s.to_string(); // \s+ needs a space
|
||
}
|
||
let b = before.as_bytes();
|
||
let digits_start = b[..digits_end].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1);
|
||
if digits_start == digits_end {
|
||
return s.to_string();
|
||
}
|
||
// \b: the digit run starts the text or follows a non-word character.
|
||
if digits_start > 0 && (b[digits_start - 1].is_ascii_alphanumeric() || b[digits_start - 1] == b'_') {
|
||
return s.to_string();
|
||
}
|
||
// A short number after a word is a set's ("… set 1 [1 of 60] "destiny-001-001.jpg" yEnc"), not a size.
|
||
let word_before = b[..digits_start].iter().rev().find(|&&c| !js::is_space(c as char)).is_some_and(|c| c.is_ascii_alphanumeric());
|
||
if digits_end - digits_start < 4 && word_before {
|
||
return s.to_string();
|
||
}
|
||
format!("{}{}", &s[..digits_start], &s[digits_end..])
|
||
}
|
||
|
||
/// The end of subjectRemainder: what's left of brackets, double spaces, trim (after the separators became single spaces).
|
||
fn finish_remainder(s: &str) -> String {
|
||
let s = if s.contains(['[', '(']) { replace(&EMPTY_BRACKETS, s, " ", true) } else { s.to_string() };
|
||
let mut out = String::with_capacity(s.len());
|
||
for c in s.chars() {
|
||
if c == ' ' && out.ends_with(' ') {
|
||
continue;
|
||
}
|
||
out.push(c);
|
||
}
|
||
js::trim(&out).to_string()
|
||
}
|
||
|
||
/// counterTotal: the subject's file counter total, first form found first.
|
||
pub fn counter_total(subject: &str) -> Option<String> {
|
||
let s = replace("ED, subject, " ", true);
|
||
if let Some(c) = COUNTERS_CAPTURE.captures(s.as_bytes()) {
|
||
let index = c.get(1).or(c.get(3)).unwrap().as_bytes();
|
||
let total = c.get(2).or(c.get(4)).unwrap().as_bytes();
|
||
return (!broken_total(index, total)).then(|| js_number_string(number(total)));
|
||
}
|
||
// Dates are no counters ("08-29-2011" read as 8 of 29).
|
||
let undated = replace(&DATES, &s, " ", true);
|
||
for re in [&*OF_COUNTER_OPEN, &*DASH_COUNTER, &*SLASH_COUNTER, &*ZERO_COUNTER] {
|
||
for c in re.captures_iter(undated.as_bytes()) {
|
||
if number(&c[1]) <= number(&c[2]) {
|
||
return Some(js_number_string(number(&c[2])));
|
||
}
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
static DATES: LazyLock<Regex> = LazyLock::new(|| re(r"\b(?:(?:19|20)[0-9]{2}[-._ ](?:0?[1-9]|1[0-2])[-._ ][0-9]{1,2}|[0-9]{1,2}[-._ ][0-9]{1,2}[-._ ](?:19|20)[0-9]{2}|[0-9]{1,2}[-._](?:19|20)[0-9]{2})\b"));
|
||
static COUNTERS_CAPTURE: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]|\(([0-9]+)/([0-9]+)\)"));
|
||
/// The "of" counter without its closing bracket (as counterTotal's pattern).
|
||
static OF_COUNTER_OPEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)[\[(]?\b(?:fileSP+)?([0-9]+)SP*ofSP*([0-9]+)\b"));
|
||
|
||
/// `String(n)` of a counter's number (integers as JS prints them).
|
||
fn js_number_string(n: f64) -> String {
|
||
if n.fract() == 0.0 && n.abs() < 1e21 {
|
||
format!("{}", n as i128)
|
||
} else {
|
||
format!("{n}")
|
||
}
|
||
}
|
||
|
||
static IMAGE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(jpe?g|png|gif|bmp|webp|tiff?)$"));
|
||
static AUDIO: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])"));
|
||
static DOCUMENT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$"));
|
||
|
||
/// An image set's name without a counter: up to the last number, without numbers.
|
||
/// imageSet (lib/release/identity.ts): the name up to the number that counts the images, digits kept.
|
||
pub fn image_set(stem: &str) -> String {
|
||
// .replace(/(\d+)\s*of\s*\d+/gi, '$1')
|
||
// Suffixes that number no image: a copy's ("…[1]"), a resolution ("…_2000x1500"), a constant one ("…_00").
|
||
let mut s = stem.as_bytes().to_vec();
|
||
for re in [&*COPY_MARK, &*RESOLUTION, &*IMAGEVENUE, &*ZERO_SUFFIX] {
|
||
s = re.replace(&s, &b""[..]).into_owned();
|
||
}
|
||
// A counter in the name counts the images: the set is the text before it ("Set 008 10of63 new0464").
|
||
if let Some(m) = NAME_COUNTER.find(&s) {
|
||
return String::from_utf8_lossy(&s[..m.start()]).into_owned();
|
||
}
|
||
// A camera's timestamp ("kurozo-2006-7-26-15-21"): the date and the time after it number no set.
|
||
let s_ts = TIMESTAMP.replace(&s, &b""[..]).into_owned();
|
||
s = s_ts;
|
||
loop {
|
||
// /\s*([_-])\s*(\d+)$/ (also spaced: "she has sensual legs 172 _ 172")
|
||
let digits = s.iter().rev().take_while(|c| c.is_ascii_digit()).count();
|
||
if digits == 0 {
|
||
break;
|
||
}
|
||
let ws = |c: &u8| matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c);
|
||
let sep_end = s.len() - digits - s[..s.len() - digits].iter().rev().take_while(|c| ws(c)).count();
|
||
if sep_end == 0 || !matches!(s[sep_end - 1], b'_' | b'-') {
|
||
break;
|
||
}
|
||
let sep = s[sep_end - 1];
|
||
let before_end = sep_end - 1 - s[..sep_end - 1].iter().rev().take_while(|c| ws(c)).count();
|
||
let n = &s[s.len() - digits..];
|
||
let before = &s[..before_end];
|
||
// /(\d+)\D*$/: the last digit run before
|
||
let Some(last) = before.iter().rposition(u8::is_ascii_digit) else { break };
|
||
let start = before[..=last].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1);
|
||
let previous = &before[start..=last];
|
||
// A repeat: the same number, or the end of the one before (054001 ends in 1).
|
||
let same = trim_zeros(n) == trim_zeros(previous) || previous.ends_with(n);
|
||
// A renumbering right after the number ("kate108lg029_28"): both count the images.
|
||
let renumbered = sep == b'_' && n[0] != b'0' && last + 1 == before.len();
|
||
let year = n.len() == 4 && (n.starts_with(b"19") || n.starts_with(b"20"));
|
||
if same || renumbered || year || n.len() >= 4 {
|
||
let cut = before.len();
|
||
s.truncate(cut);
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
let set = match s.iter().rposition(u8::is_ascii_digit) {
|
||
Some(last) => &s[..s[..=last].iter().rposition(|c| !c.is_ascii_digit()).map_or(0, |i| i + 1)],
|
||
None => return String::new(),
|
||
};
|
||
// A number of 5+ digits left in it is an id, per image ("sss-70556-3474182-23-10825418-pc"): no set.
|
||
if set.split(|c| !c.is_ascii_digit()).any(|run| trim_zeros(run).len() >= 5) {
|
||
return String::new();
|
||
}
|
||
String::from_utf8_lossy(set).into_owned()
|
||
}
|
||
|
||
/// Number(digits) equality for digit strings: without leading zeros (all zeros: "0").
|
||
fn trim_zeros(d: &[u8]) -> &[u8] {
|
||
let t = d.iter().position(|&c| c != b'0').map_or(&d[d.len().saturating_sub(1)..], |i| &d[i..]);
|
||
t
|
||
}
|
||
|
||
static TIMESTAMP: LazyLock<Regex> = LazyLock::new(|| re(r"[-_. ]?(?:19|20)[0-9]{2}[-._ ](?:0?[1-9]|1[0-2])[-._ ][0-9]{1,2}(?:[-._ ][0-9]{1,2})*$"));
|
||
static COPY_MARK: LazyLock<Regex> = LazyLock::new(|| re(r"(?:SP*[\[(][0-9]+[\])])+$"));
|
||
static RESOLUTION: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)[_-][0-9]{3,4}(?:x[0-9]{3,4}|px)$"));
|
||
static IMAGEVENUE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)_123_[0-9]+lo$"));
|
||
static ZERO_SUFFIX: LazyLock<Regex> = LazyLock::new(|| re(r"[_-]0+$"));
|
||
static NAME_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)([0-9]+)SP*ofSP*[0-9]+|[\[(]?SP*[0-9]+SP*/SP*[0-9]+SP*[\])]?"));
|
||
|
||
fn image_prefix(stem: &str) -> String {
|
||
let b = stem.as_bytes();
|
||
let cut = match b.iter().rposition(u8::is_ascii_digit) {
|
||
Some(last) => b[..=last].iter().rposition(|c| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0),
|
||
None => b.len(),
|
||
};
|
||
let head = &stem[..cut];
|
||
let mut out = String::with_capacity(head.len());
|
||
let mut sep = false;
|
||
for c in head.chars() {
|
||
if c.is_ascii_digit() || js::is_space(c) || matches!(c, '.' | '_' | '-') {
|
||
sep = true;
|
||
} else {
|
||
if sep && !out.is_empty() {
|
||
out.push(' ');
|
||
}
|
||
out.push(c);
|
||
sep = false;
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
fn is_random_token(token: &str) -> bool {
|
||
let t = token.strip_prefix('[').unwrap_or(token);
|
||
let t = t.strip_suffix(']').unwrap_or(t);
|
||
if len16(t) < 6 {
|
||
return false;
|
||
}
|
||
let digit = t.bytes().any(|b| b.is_ascii_digit());
|
||
if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||
return true;
|
||
}
|
||
digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-'))
|
||
}
|
||
|
||
pub fn is_post_token(remainder: &str) -> bool {
|
||
let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect();
|
||
let n = len16(&compact);
|
||
(6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric())
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
static NAME_LIKE: LazyLock<Regex> = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}"));
|
||
|
||
pub fn names_release(remainder: &str) -> bool {
|
||
let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect();
|
||
name_like(words.join(" ").as_bytes())
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct Release {
|
||
pub key: String,
|
||
pub name: String,
|
||
pub source: &'static str,
|
||
/// The [i/M] total as parsed (0 is falsy in the TypeScript).
|
||
pub file_total: Option<i64>,
|
||
}
|
||
|
||
impl Release {
|
||
pub fn file_total_truthy(&self) -> Option<i64> {
|
||
self.file_total.filter(|&n| n != 0)
|
||
}
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
static FILE_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]"));
|
||
|
||
fn hash(parts: &[&str]) -> String {
|
||
sha1_hex(&parts.join("\n"))
|
||
}
|
||
|
||
/// A name that says what it is: 12+ letters and digits, two words of 3+ letters (`describes`).
|
||
fn describes(stem: &str) -> bool {
|
||
let b = stem.as_bytes();
|
||
if b.iter().filter(|c| c.is_ascii_alphanumeric()).count() < 12 {
|
||
return false;
|
||
}
|
||
// [A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}
|
||
let mut i = 0;
|
||
while i < b.len() {
|
||
let letters = b[i..].iter().take_while(|c| c.is_ascii_alphabetic()).count();
|
||
if letters == 0 {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
let end = i + letters;
|
||
if letters >= 3 {
|
||
let gap = b[end..].iter().take_while(|c| !c.is_ascii_alphanumeric()).count();
|
||
if gap > 0 && b[end + gap..].iter().take_while(|c| c.is_ascii_alphabetic()).count() >= 3 {
|
||
return true;
|
||
}
|
||
}
|
||
i = end;
|
||
}
|
||
false
|
||
}
|
||
|
||
const TAG_WORDS: [&str; 21] = [
|
||
"www", "http", "https", "com", "net", "org", "info", "the", "and", "presents", "posted", "post", "powered", "yenc", "part", "rar", "par2", "nzb", "voor", "for", "from",
|
||
];
|
||
|
||
/// isPosterTag: the text outside the quotes shares no word with the file's stem.
|
||
pub fn is_poster_tag(remainder: &str, stem: &str, counted: bool) -> bool {
|
||
let lower = stem.to_lowercase();
|
||
let words: Vec<&str> = lower.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())).filter(|w| !w.is_empty()).collect();
|
||
let compact: String = words.concat();
|
||
let whole: std::collections::HashSet<&str> = words.iter().copied().collect();
|
||
// A per-post id, also inside a word ("FTD772544"), under a counter; without one the board's or the dump's.
|
||
let b = remainder.as_bytes();
|
||
if counted && b.windows(5).any(|w| w.iter().all(u8::is_ascii_digit)) {
|
||
return false;
|
||
}
|
||
let rem = remainder.to_lowercase();
|
||
let (mut significant, mut shared) = (0, 0);
|
||
for word in rem.split(|c: char| !(c.is_ascii_lowercase() || c.is_ascii_digit())) {
|
||
if !word.is_empty() && word.bytes().all(|c| c.is_ascii_digit()) {
|
||
if word.len() >= 2 && whole.contains(word) {
|
||
return false;
|
||
}
|
||
continue;
|
||
}
|
||
if word.len() < 3 || TAG_WORDS.contains(&word) {
|
||
continue;
|
||
}
|
||
significant += 1;
|
||
if if word.len() >= 4 { compact.contains(word) } else { whole.contains(word) } {
|
||
shared += 1;
|
||
}
|
||
}
|
||
// Without a counter, a tag of 4+ words may share one with the stem ("German Board - [Usenet4ever.info] und [SecretUsenet.com]").
|
||
// With one, that's a name ("[ MacGyver.S01E03.DVDRip.XviD-MEDiEVAL ]-[04/38] - "sample.macgy.s01e03-med.avi"").
|
||
shared == 0 || (shared == 1 && significant >= 4 && !counted)
|
||
}
|
||
|
||
/// releaseIdentity for a named (not weak) file: never null.
|
||
/// randomPoster (lib/release/identity.ts): a poster whose own name, before the "@", is a random token.
|
||
pub fn random_poster(from: &str) -> bool {
|
||
let address = match from.find('<').and_then(|open| from[open + 1..].find('>').map(|close| &from[open + 1..open + 1 + close])) {
|
||
Some(address) => address,
|
||
None => js::trim(from).split(' ').next().unwrap_or(""),
|
||
};
|
||
// Separators don't make a token less random.
|
||
let local: String = address.split_once('@').map_or(address, |(local, _)| local).chars().filter(|c| !matches!(c, '-' | '_' | '.')).collect();
|
||
looks_random(&local)
|
||
}
|
||
|
||
/// RUN_ON (lib/release/identity.ts): an archive's or par2's extension.
|
||
static RUN_ON: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(?:part[0-9]+\.rar|vol[0-9]+[+-][0-9]+\.par2|par2|rar|r[0-9]{2,3}|7z(?:\.[0-9]{3})?|zip|nzb|nfo|sfv)"));
|
||
|
||
/// runOn (lib/release/identity.ts): a quoted name that runs on past its extension into a title, cut there.
|
||
pub fn run_on(name: &str) -> &str {
|
||
let b = name.as_bytes();
|
||
let ascii = b.len() >= 4 && b[..4].is_ascii();
|
||
for m in RUN_ON.find_iter(b) {
|
||
let rest = &name[m.end()..];
|
||
if rest.is_empty() {
|
||
return name;
|
||
}
|
||
// Not a bare dash ("….READ.NFO-THETA.rar").
|
||
let spaced = rest.starts_with(js::is_space);
|
||
let repeats = ascii && rest.len() >= 4 && rest.as_bytes()[..4].eq_ignore_ascii_case(&b[..4]);
|
||
if spaced || repeats {
|
||
return &name[..m.end()];
|
||
}
|
||
}
|
||
name
|
||
}
|
||
|
||
/// sizeAfterName (lib/release/identity.ts): the subject without a number after the file's unquoted name at its end.
|
||
pub fn size_after_name(subject: &str, name: &str) -> String {
|
||
let end = subject.trim_end_matches(js::is_space);
|
||
let before_digits = end.trim_end_matches(|c: char| c.is_ascii_digit());
|
||
if before_digits.len() == end.len() {
|
||
return subject.to_string();
|
||
}
|
||
// /\s*-\s*$/
|
||
let t = before_digits.trim_end_matches(js::is_space);
|
||
let before = match t.strip_suffix('-') {
|
||
Some(t) => t.trim_end_matches(js::is_space),
|
||
None => return subject.to_string(),
|
||
};
|
||
if !name.is_empty() && before.ends_with(name) && !before.ends_with(&format!("\"{name}\"")) {
|
||
before.to_string()
|
||
} else {
|
||
subject.to_string()
|
||
}
|
||
}
|
||
|
||
pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||
let poster = if random_poster(poster) { "" } else { poster };
|
||
// The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal).
|
||
let post_total = file_counter_total(subject);
|
||
let stem = file_stem(run_on(name));
|
||
// A name that isn't quoted (unquoted_name) is taken out as the quotes would have been.
|
||
// The subject here has its spaces collapsed; the name may not.
|
||
// Other quoted text stays: it names the release (">>> CREATIVE COMMONS NZB <<< "0190 alarm 5.0 " - File 1 of 9: …").
|
||
// A size after a name that isn't quoted, at the end ("[27/70] - 2407.jpg - 28").
|
||
let subject = size_after_name(subject, name);
|
||
let subject = subject.as_str();
|
||
let quoted = format!("\"{name}\"");
|
||
let without = if subject.contains("ed) {
|
||
subject.replacen("ed, " ", 1)
|
||
} else {
|
||
let collapsed = js::collapse_spaces(name);
|
||
let collapsed = js::trim(&collapsed);
|
||
if subject.contains(name) { subject.replacen(name, " ", 1) } else { subject.replacen(collapsed, " ", 1) }
|
||
};
|
||
// The name repeated elsewhere ("(NELET11.par2) [01/58] - "NELET11.par2"").
|
||
let without = if name.encode_utf16().count() >= 5 { without.replace(name, " ") } else { without };
|
||
let named = without.replace('"', " ");
|
||
// A file's size in brackets after its counter ("[26/32] - [4929] "…part24.rar"").
|
||
let named = replace(&SIZE_AFTER_COUNTER, &named, "$1", true);
|
||
// A per-file hex id in brackets ("… [1f1793f68d5321] [newzNZB] [01/22]").
|
||
let named = replace(&HEX_ID, &named, " ", true);
|
||
// Counted without the file name: "shyla-shy-09-0195.jpg hardcore …" is no "9 of 195".
|
||
let total = counter_total(&named);
|
||
let file_total = total.as_ref().and_then(|t| t.parse::<i64>().ok());
|
||
// Without a counter a host and a sponsor name no upload ("(www.Thunder-News.org) >XXX< <Sponsored by Secretusenet>").
|
||
let remainder = subject_remainder(&named);
|
||
let remainder = if total.is_none() {
|
||
let bare = replace(&BOILERPLATE, &remainder, " ", true);
|
||
js::trim(&js::collapse_spaces(&bare)).to_string()
|
||
} else {
|
||
remainder
|
||
};
|
||
// A document or comic is a release of its own.
|
||
if test(&DOCUMENT, name) {
|
||
let shown = replace(&DOCUMENT, &stem, "", false);
|
||
let name = if shown.is_empty() { stem.clone() } else { shown };
|
||
return Release { key: hash(&["stem", poster, &stem]), name, source: "stem", file_total };
|
||
}
|
||
let image = test(&IMAGE, name);
|
||
// The poster's constant tag outside the quotes: the file's stem says which upload it is (not for images and audio).
|
||
// Not for numbered items of a set ("003 Defining Tools with JSON Schema.mp4"): /^\d{1,3}[ ._-]/.
|
||
// /^(?:[A-Z]{1,5}\d{2,6}[-_ ])?\d{1,3}[ ._-]/: also after a disc code ("SC8367-12 - …").
|
||
let numbered = {
|
||
let b = stem.as_bytes();
|
||
let index_at = |at: usize| {
|
||
let digits = b[at..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
(1..=3).contains(&digits) && matches!(b.get(at + digits), Some(b' ' | b'.' | b'_' | b'-'))
|
||
};
|
||
let letters = b.iter().take_while(|c| c.is_ascii_uppercase()).count();
|
||
let code_digits = b[letters..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let code = (1..=5).contains(&letters) && (2..=6).contains(&code_digits) && matches!(b.get(letters + code_digits), Some(b'-' | b'_' | b' '));
|
||
let date = date_led(b);
|
||
(index_at(0) || (code && index_at(letters + code_digits + 1))) && !date
|
||
};
|
||
// A hashed stem under text without a counter is the upload, the text a tag ("Seit 1753 Tagen - usenet4ever.info - …").
|
||
let hashed = file_total.is_none() && test(&HASHED, &stem);
|
||
// A dated stem says which upload it is, short as it is ("09-30-2008_CTC.r00").
|
||
let dated = date_led(stem.as_bytes());
|
||
let tag = hashed || (!image && !numbered && !test(&AUDIO, name) && (describes(&stem) || dated) && !looks_random(&stem) && is_poster_tag(&remainder, &stem, total.is_some()));
|
||
let prefix = if image { image_prefix(&stem) } else { String::new() };
|
||
// An image set's subject may be one word, the model's name: /^[A-Za-z]{3,}$/.
|
||
let one_word = image && remainder.len() >= 3 && remainder.bytes().all(|c| c.is_ascii_alphabetic());
|
||
if !tag && (names_release(&remainder) || one_word) {
|
||
// Without a counter an image's set: its name up to its last number, digits kept; none without numbers.
|
||
let bound = total.clone().unwrap_or_else(|| if image { image_set(&stem) } else { String::new() });
|
||
return Release { key: hash(&["subject", poster, &remainder, &bound]), name: remainder, source: "subject", file_total };
|
||
}
|
||
if let Some(total) = post_total.filter(|&n| n != 0 && !tag) {
|
||
if is_post_token(&remainder) {
|
||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||
}
|
||
}
|
||
// A scene music post's index files ("00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg") under a bare counter go
|
||
// with its tracks: the artist and the total.
|
||
if let Some(total) = post_total.filter(|&n| n != 0) {
|
||
if stem.len() > 2 && stem.starts_with("00") && matches!(stem.as_bytes()[2], b'-' | b'_' | b' ' | b'.') {
|
||
if let Some(c) = SCENE_TRACK.captures(stem.as_bytes()) {
|
||
let artist = String::from_utf8(c[1].to_vec()).unwrap();
|
||
return Release { key: hash(&["stem", poster, &artist, &total.to_string()]), name: artist, source: "stem", file_total };
|
||
}
|
||
}
|
||
// And its par2 and sfv under the release's own name ("blaise_mc-in_hellstill_3-ep-web-2022-known.par2").
|
||
// Only a track post's lower-case name: a rar upload groups by its full name.
|
||
if !test(&AUDIO, name) && !stem.chars().any(|c| c.is_ascii_uppercase() || js::is_space(c)) {
|
||
if let Some(c) = SCENE_RELEASE.captures(stem.as_bytes()) {
|
||
let artist = String::from_utf8(c[1].to_vec()).unwrap();
|
||
return Release { key: hash(&["stem", poster, &artist, &total.to_string()]), name: artist, source: "stem", file_total };
|
||
}
|
||
}
|
||
}
|
||
if image {
|
||
// Keyed by its set when its name has one ("diana-040-001.jpg").
|
||
let name = if prefix.is_empty() { stem.clone() } else { prefix.clone() };
|
||
let set = image_set(&stem);
|
||
let key = if set.is_empty() { hash(&["stem", poster, &name]) } else { hash(&["stem", poster, &set]) };
|
||
return Release { key, name, source: "stem", file_total };
|
||
}
|
||
// An audiobook's or album's tracks without a name around them: by their name without its numbers.
|
||
if test(&AUDIO, name) {
|
||
// Scene tracks under a bare counter ("02-repiet_-_dont_think_twice_(extended_mix).mp3"): the artist's album.
|
||
if let Some(total) = post_total.filter(|&n| n != 0) {
|
||
if let Some(c) = SCENE_TRACK.captures(stem.as_bytes()) {
|
||
let artist = String::from_utf8(c[1].to_vec()).unwrap();
|
||
return Release { key: hash(&["stem", poster, &artist, &total.to_string()]), name: artist, source: "stem", file_total };
|
||
}
|
||
}
|
||
let tracks = image_prefix(&stem);
|
||
if !tracks.is_empty() {
|
||
return Release { key: hash(&["stem", poster, &tracks]), name: tracks, source: "stem", file_total };
|
||
}
|
||
}
|
||
// A season's episodes under a bare counter ("Pokemon - S02E01 - Pallet Party Panic.mkv"): per show, season and total.
|
||
if let Some(total) = post_total.filter(|&n| n != 0) {
|
||
if test(&VIDEO, name) {
|
||
if let Some(c) = EPISODE.captures(stem.as_bytes()) {
|
||
let show = String::from_utf8(c[1].to_vec()).unwrap();
|
||
let season = format!("{} S{:0>2}", js::trim(&show), std::str::from_utf8(&c[2]).unwrap());
|
||
return Release { key: hash(&["stem", poster, &season.to_lowercase(), &total.to_string()]), name: season, source: "stem", file_total };
|
||
}
|
||
}
|
||
}
|
||
// A numbered volume as the extension, within the counter's total ("DN.VPL PCK.1508" of 2195): one upload.
|
||
let base = match (VOLUME_NUMBER.captures(stem.as_bytes()), post_total.filter(|&n| n != 0)) {
|
||
(Some(c), Some(total)) if number(&c[2]) <= total as f64 => String::from_utf8(c[1].to_vec()).unwrap(),
|
||
_ => stem.clone(),
|
||
};
|
||
// Without case and a scene index's "00-" ("00-laura_van_dam_-_…-talion.m3u" beside "Laura_van_Dam_-_…-TALiON.par2").
|
||
let lower = base.to_lowercase();
|
||
let b = lower.as_bytes();
|
||
let index = if b.starts_with(b"00") { 2 + b[2..].iter().take_while(|c| matches!(c, b'-' | b'_' | b' ' | b'.')).count() } else { 0 };
|
||
let key_stem = if index > 2 { &lower[index..] } else { &lower[..] };
|
||
Release { key: hash(&["stem", poster, key_stem]), name: stem, source: "stem", file_total }
|
||
}
|
||
|
||
static HEX_ID: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\[[0-9a-f]{8,}\]"));
|
||
static VIDEO: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(mkv|mp4|avi|m4v|ts|wmv|mov)$"));
|
||
static EPISODE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)^(.+?)[ ._-]+S([0-9]{1,2})E[0-9]{1,3}"));
|
||
static SIZE_AFTER_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"(\[[0-9]+/[0-9]+\])SP*-SP*\[[0-9]+\]"));
|
||
/// A host and a sponsor in a subject's remainder (BOILERPLATE in lib/release/identity.ts).
|
||
static BOILERPLATE: LazyLock<Regex> =
|
||
LazyLock::new(|| re(r"(?i)\(?\bwww(?: [A-Za-z0-9]+)*? (?:org|com|net|info|nl|de|eu|to|biz|us)\b\)?|<?\bsponsored by [A-Za-z0-9]+>?"));
|
||
/// Also one long hex run ("6c57319fbae505679737bd46a76193a1-u4e.part6.rar").
|
||
static HASHED: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)^(?:[0-9a-f]{8,}(?:_[0-9a-f]{6,}){2,}|[0-9a-f]{24,})(?:-[0-9a-z]+)?$"));
|
||
static SCENE_TRACK: LazyLock<Regex> = LazyLock::new(|| re(r"^[0-9]{1,3}[-_ .]+(.+?)(?:_-_| - |-)"));
|
||
/// SCENE_RELEASE (lib/release/identity.ts): a scene music release's name.
|
||
static SCENE_RELEASE: LazyLock<Regex> = LazyLock::new(|| {
|
||
re(r"(?i)^(.+?)(?:_-_| - |-).*-(?:web|cd[a-z]?|vinyl|vls|sat|cable|fm|dab|single|ep|lp|promo|bootleg)-.*(?:19|20)[0-9]{2}(?:-[A-Za-z0-9_]+)?$")
|
||
});
|
||
static VOLUME_NUMBER: LazyLock<Regex> = LazyLock::new(|| re(r"^(.*[^0-9. ])[. ]([0-9]{1,5})\.?$"));
|
||
|
||
static GENERIC: LazyLock<Regex> = LazyLock::new(|| {
|
||
re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$")
|
||
});
|
||
|
||
fn is_generic_name(name: &str) -> bool {
|
||
test(&GENERIC, js::trim(name))
|
||
}
|
||
|
||
/// looksRandom.
|
||
pub fn looks_random(name: &str) -> bool {
|
||
// name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||
let name = match name.rfind('.') {
|
||
Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i],
|
||
_ => name,
|
||
};
|
||
if is_generic_name(name) {
|
||
return true;
|
||
}
|
||
let b = name.as_bytes();
|
||
let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit());
|
||
let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit());
|
||
if b.len() >= 16 && hex(b) {
|
||
return true;
|
||
}
|
||
if let Some(dot) = name.find('.') {
|
||
let (h, d) = (&b[..dot], &b[dot + 1..]);
|
||
if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) {
|
||
return true;
|
||
}
|
||
}
|
||
if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) {
|
||
return true;
|
||
}
|
||
{
|
||
let core = name.strip_suffix(" n").unwrap_or(name).as_bytes();
|
||
if core.len() >= 6 && digits(core) {
|
||
return true;
|
||
}
|
||
}
|
||
let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count();
|
||
let n_digit = count(u8::is_ascii_digit);
|
||
let n_lower = count(u8::is_ascii_lowercase);
|
||
let n_upper = count(u8::is_ascii_uppercase);
|
||
let alnum = b.iter().all(|c| c.is_ascii_alphanumeric());
|
||
if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 {
|
||
return true;
|
||
}
|
||
if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 {
|
||
return true;
|
||
}
|
||
// 51294P1CUKR2R35X: capitals and digits only, switching between them all over the token
|
||
// (COD4MW2REMASTERED switches 4 times, BEATLES1964REMASTER twice).
|
||
if b.len() >= 12 && b.iter().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) && b.windows(2).filter(|w| w[0].is_ascii_digit() != w[1].is_ascii_digit()).count() >= 6 {
|
||
return true;
|
||
}
|
||
if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 {
|
||
return false;
|
||
}
|
||
n_digit > 0 || n_upper >= 6
|
||
}
|
||
|
||
pub fn is_random_file(name: &str) -> bool {
|
||
looks_random(&file_stem(name))
|
||
}
|
||
|
||
static SEVEN_SPLIT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$"));
|
||
static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||
|
||
pub fn is_par2(name: &str) -> bool {
|
||
test(&PAR2, name)
|
||
}
|
||
|
||
pub fn seven_zip_volume(name: &str) -> Option<i64> {
|
||
if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) {
|
||
return js::parse_int(std::str::from_utf8(&c[1]).unwrap());
|
||
}
|
||
if test(&SEVEN, name) {
|
||
Some(0)
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
|
||
pub fn is_archive_start(name: &str) -> bool {
|
||
if test(&BIN, name) {
|
||
return true;
|
||
}
|
||
if matches!(seven_zip_volume(name), Some(0) | Some(1)) {
|
||
return true;
|
||
}
|
||
test(&PART1, name) || (test(&RAR, name) && !test(&PART, name))
|
||
}
|
||
|
||
pub fn is_later_volume(name: &str) -> bool {
|
||
if let Some(c) = PART.captures(name.as_bytes()) {
|
||
return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1;
|
||
}
|
||
test(&OLD_RAR, name) || test(&PAR2_VOL, name)
|
||
}
|
||
|
||
static TOOLS: LazyLock<Vec<(&'static str, Regex)>> = LazyLock::new(|| {
|
||
vec![
|
||
("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")),
|
||
("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")),
|
||
("ngPost", re(r"^[0-9a-f]{32}@ngPost$")),
|
||
("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")),
|
||
("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")),
|
||
("JBinUp", re(r"@JBin(?:Up|Down)\.local$")),
|
||
("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")),
|
||
("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")),
|
||
("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")),
|
||
]
|
||
});
|
||
|
||
/// uploadTool with the regexes, as in the TypeScript (the reference for upload_tool's test).
|
||
#[allow(dead_code)]
|
||
pub fn upload_tool_re(message_id: &str) -> &'static str {
|
||
if message_id.is_empty() {
|
||
return "other";
|
||
}
|
||
for (tool, pattern) in TOOLS.iter() {
|
||
if test(pattern, message_id) {
|
||
return tool;
|
||
}
|
||
}
|
||
"other"
|
||
}
|
||
|
||
fn all(s: &[u8], f: impl Fn(u8) -> bool) -> bool {
|
||
s.iter().all(|&c| f(c))
|
||
}
|
||
|
||
fn lower_hex(c: u8) -> bool {
|
||
c.is_ascii_digit() || (b'a'..=b'f').contains(&c)
|
||
}
|
||
|
||
/// `(?:[A-Z][a-z]){12}-\d{13}@`: nyuu's prefix (38 bytes, then '@').
|
||
fn nyuu_prefix(b: &[u8]) -> bool {
|
||
b.len() > 38 && (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[24] == b'-' && all(&b[25..38], |c| c.is_ascii_digit()) && b[38] == b'@'
|
||
}
|
||
|
||
/// The posting tool of a Message-ID (TOOLS, in order, by hand: they ran for every per-article file).
|
||
pub fn upload_tool(message_id: &str) -> &'static str {
|
||
let b = message_id.as_bytes();
|
||
if b.is_empty() {
|
||
return "other";
|
||
}
|
||
if nyuu_prefix(b) && b.len() > 39 {
|
||
// nyuu: "@nyuu" exactly; nyuu-host: any host (`.+`: no line terminators).
|
||
if &b[39..] == b"nyuu" {
|
||
return "nyuu";
|
||
}
|
||
if !b[39..].iter().any(|&c| c == b'\n' || c == b'\r') && !message_id[39..].contains(['\u{2028}', '\u{2029}']) {
|
||
return "nyuu-host";
|
||
}
|
||
}
|
||
let hex32 = b.len() > 33 && all(&b[..32], lower_hex) && b[32] == b'@';
|
||
if hex32 {
|
||
let host = &b[33..];
|
||
if host == b"ngPost" {
|
||
return "ngPost";
|
||
}
|
||
if host.len() == 37 && all(&host[..32], lower_hex) && &host[32..] == b".yuun" {
|
||
return "yuun";
|
||
}
|
||
if all(host, |c| c.is_ascii_alphanumeric() || c == b'.' || c == b'-') {
|
||
return "hex32";
|
||
}
|
||
}
|
||
if message_id.ends_with("@JBinUp.local") || message_id.ends_with("@JBinDown.local") {
|
||
return "JBinUp";
|
||
}
|
||
// (?i)^part\d+of\d+\.
|
||
if b.len() > 4 && eq_ci(&b[..4], b"part") {
|
||
let d1 = b[4..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let of = 4 + d1;
|
||
if d1 > 0 && b.len() > of + 2 && eq_ci(&b[of..of + 2], b"of") {
|
||
let d2 = b[of + 2..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if d2 > 0 && b.get(of + 2 + d2) == Some(&b'.') {
|
||
return "partNofM";
|
||
}
|
||
}
|
||
}
|
||
// ^\d{13}\$\d+\$\d+@localhost$
|
||
if let Some(rest) = message_id.strip_suffix("@localhost") {
|
||
let r = rest.as_bytes();
|
||
let parts: Vec<&[u8]> = r.split(|&c| c == b'$').collect();
|
||
if parts.len() == 3 && parts[0].len() == 13 && parts.iter().all(|p| !p.is_empty() && all(p, |c| c.is_ascii_digit())) {
|
||
return "ts-dollar";
|
||
}
|
||
}
|
||
// ^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$
|
||
if let Some(at) = b.iter().position(|&c| c == b'@') {
|
||
let (local, host) = (&b[..at], &b[at + 1..]);
|
||
if (10..=40).contains(&local.len()) && all(local, |c| c.is_ascii_alphanumeric()) {
|
||
if let Some(dot) = host.iter().position(|&c| c == b'.') {
|
||
let (name, tld) = (&host[..dot], &host[dot + 1..]);
|
||
if (3..=12).contains(&name.len()) && (2..=6).contains(&tld.len()) && all(name, |c| c.is_ascii_alphanumeric()) && all(tld, |c| c.is_ascii_alphanumeric()) {
|
||
return "random";
|
||
}
|
||
}
|
||
}
|
||
}
|
||
"other"
|
||
}
|
||
|
||
/// [A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,} anywhere.
|
||
fn name_like(b: &[u8]) -> bool {
|
||
let sep = |c: u8| matches!(c, b'.' | b'_' | b' ' | b'-' | b',' | b'&' | b':' | b';' | b'\\' | b'/' | b'+');
|
||
(2..b.len()).any(|k| {
|
||
if !(sep(b[k]) && b[k - 1].is_ascii_alphabetic() && b[k - 2].is_ascii_alphabetic()) {
|
||
return false;
|
||
}
|
||
let m = k + b[k..].iter().take_while(|&&c| sep(c)).count();
|
||
m + 2 <= b.len() && b[m].is_ascii_alphanumeric() && b[m + 1].is_ascii_alphanumeric()
|
||
})
|
||
}
|
||
|
||
/// /^\d{1,2}[-._]\d{1,2}[-._](?:19|20)\d{2}(?!\d)/: a stem that starts with a date ("09-30-2008_BAK_JRS").
|
||
fn date_led(b: &[u8]) -> bool {
|
||
let run = |at: usize| b.get(at..).map_or(0, |s| s.iter().take_while(|c| c.is_ascii_digit()).count());
|
||
let sep = |at: usize| matches!(b.get(at), Some(b'-' | b'.' | b'_'));
|
||
let d1 = run(0);
|
||
let d2 = if (1..=2).contains(&d1) && sep(d1) { run(d1 + 1) } else { 0 };
|
||
let y = d1 + 1 + d2 + 1;
|
||
(1..=2).contains(&d2) && sep(d1 + 1 + d2) && run(y) == 4 && matches!(&b[y..y + 2], b"19" | b"20")
|
||
}
|
||
|
||
/// A total zero-padded wider than its index ("[01/010]"): a posting tool's mistake, no counter (brokenTotal).
|
||
fn broken_total(index: &[u8], total: &[u8]) -> bool {
|
||
total.len() > 1 && total[0] == b'0' && total.len() > index.len()
|
||
}
|
||
|
||
/// The second number of the first \[(\d+)/(\d+)\].
|
||
fn file_counter_total(s: &str) -> Option<i64> {
|
||
let b = s.as_bytes();
|
||
for (i, &c) in b.iter().enumerate() {
|
||
if c != b'[' {
|
||
continue;
|
||
}
|
||
let a = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let slash = i + 1 + a;
|
||
if a == 0 || b.get(slash) != Some(&b'/') {
|
||
continue;
|
||
}
|
||
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
|
||
if broken_total(&b[i + 1..slash], &b[slash + 1..slash + 1 + d]) {
|
||
return None;
|
||
}
|
||
return js::parse_int(&s[slash + 1..slash + 1 + d]);
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// explored: a release in the sampled share of a rule's drops.
|
||
pub fn explored(key: &str, percent: f64) -> bool {
|
||
if percent <= 0.0 {
|
||
return false;
|
||
}
|
||
let d = Md5::digest(key.as_bytes());
|
||
let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]);
|
||
((n % 10_000) as f64) < percent * 100.0
|
||
}
|
||
|
||
static SIDECAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$"));
|
||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||
|
||
pub struct JudgedFile<'a> {
|
||
pub name: &'a str,
|
||
pub total_parts: u32,
|
||
pub groups: &'a [String],
|
||
pub first_message_id: &'a str,
|
||
}
|
||
|
||
pub enum Verdict {
|
||
Keep,
|
||
Drop { rule: &'static str, key: String, sample: bool },
|
||
}
|
||
|
||
/// The rules a lease brings (lib/rules Rules): what the worker drops.
|
||
#[derive(Clone, Debug)]
|
||
pub struct Rules {
|
||
pub version: i64,
|
||
/// Indexed groups and their priority (owner rule).
|
||
pub priorities: rustc_hash::FxHashMap<String, u32>,
|
||
/// "tool\tgroup"
|
||
pub dropped_signatures: rustc_hash::FxHashSet<String>,
|
||
pub dropped_tools: rustc_hash::FxHashSet<String>,
|
||
pub per_article_max_parts: u32,
|
||
pub sample_percent: f64,
|
||
pub drop_executables: bool,
|
||
}
|
||
|
||
impl Rules {
|
||
/// DEFAULT_RULES with these priorities and no dropped signatures (edge:bench's).
|
||
pub fn defaults(priorities: rustc_hash::FxHashMap<String, u32>) -> Self {
|
||
Rules {
|
||
version: 1,
|
||
priorities,
|
||
dropped_signatures: Default::default(),
|
||
dropped_tools: ["ngPost", "hex32", "yuun", "JBinUp", "random"].iter().map(|t| t.to_string()).collect(),
|
||
per_article_max_parts: 3,
|
||
sample_percent: 1.0,
|
||
drop_executables: true,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// RuleSet.judge.
|
||
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
||
let sample = explored(release_key, rules.sample_percent);
|
||
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||
if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||
let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
|
||
return Verdict::Drop { rule: "executable", key, sample };
|
||
}
|
||
if files.iter().any(|f| !is_random_file(f.name)) {
|
||
return Verdict::Keep;
|
||
}
|
||
let first = &files[0];
|
||
let tool = upload_tool(first.first_message_id);
|
||
let group = first.groups.first().map(String::as_str).unwrap_or("");
|
||
if rules.dropped_tools.contains(tool) || rules.dropped_signatures.contains(&format!("{tool}\t{group}")) {
|
||
return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample };
|
||
}
|
||
if files.iter().all(|f| f.total_parts <= rules.per_article_max_parts) {
|
||
return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample };
|
||
}
|
||
Verdict::Keep
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn stems() {
|
||
assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x");
|
||
assert_eq!(file_stem("x.part07.rar"), "x");
|
||
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||
assert_eq!(file_stem("a.7z.002"), "a");
|
||
}
|
||
|
||
#[test]
|
||
fn remainders() {
|
||
assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release");
|
||
assert!(names_release("Some Release"));
|
||
assert!(!names_release("elys de81bcabef5faf6a"));
|
||
}
|
||
|
||
#[test]
|
||
fn by_hand_as_the_regexes() {
|
||
let names = [
|
||
"x.mkv.vol03+04.par2", "x.part07.rar", "Movie.H.264.mkv.001", "a.7z.002", "a.7z", "b.zip.001", "B.ZIP", "c.r00", "c.r1234",
|
||
"d.vol1+2.par2", "d.vol+2.par2", ".par2", "e.tar.gz", "f.mkv", "g.m3u8", "h.bin.001", " spaced.rar ", "i.PART01.RAR", "j.part.rar",
|
||
"k.nfo.txt", "noext", "ünï.cödé.mkv.001", "l.ts.123", "m.r12.par2", "n.vol12-34.PAR2", "o.7z.1234", "p.123", "q.r1", ".rar.rar",
|
||
];
|
||
for n in names {
|
||
assert_eq!(file_stem(n), file_stem_re(n), "{n}");
|
||
}
|
||
let subjects = [
|
||
r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#,
|
||
r#"[1/3] - "a.par2" yEnc (1/1) 123 "#,
|
||
r#"yEnc "x" yenc yEncx (1/2)"#,
|
||
r#"Show – S01E01 | 1080p : x "a" "b"#,
|
||
r#"name (12/3x) [4/] "q" YENC 12 34"#,
|
||
" lead\u{a0}nbsp yEnc\u{3000}99 ",
|
||
r#"elys-de81bcabef5faf6a - [Nyh5Q07Y] - "x.mkv" yEnc"#,
|
||
"a_yEnc b yEnc_c yEnc",
|
||
"yEnc yEnc 5",
|
||
"x yEnc 12 yEnc",
|
||
"",
|
||
];
|
||
for s in subjects {
|
||
assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn tools_by_hand_as_the_regexes() {
|
||
let ids = [
|
||
"FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@news.example",
|
||
"b94268bc702040f7b811d5318f727e4f@ngPost", "b94268bc702040f7b811d5318f727e4f@b94268bc702040f7b811d5318f727e4f.yuun", "b94268bc702040f7b811d5318f727e4f@host-1.example",
|
||
"B94268bc702040f7b811d5318f727e4f@ngPost", "x@JBinUp.local", "y@JBinDown.local", "part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local", "PART12OF3.x",
|
||
"1790923856877$1729905757859$1@localhost", "179092385687$1$1@localhost", "t3Y4mV92YCp0McxDc5@zLZlsw.RYqes", "t3Y4mV92YC@zL.RYqes", "t3Y4mV92YCp0McxDc5@zLZlsw.RY.es",
|
||
"", "a@b", "b94268bc702040f7b811d5318f727e4f@", "b94268bc702040f7b811d5318f727e4f@host_x",
|
||
];
|
||
for id in ids {
|
||
assert_eq!(upload_tool(id), upload_tool_re(id), "{id}");
|
||
}
|
||
for t in ["Some Release", "a b", "ab-cd", "ab - c1", "ab.c", "x ab_12", "12 ab", "ab "] {
|
||
assert_eq!(name_like(t.as_bytes()), test(&NAME_LIKE, t), "{t}");
|
||
}
|
||
for t in ["[01/45] x", "x [1/] [2/30]", "[a/1] [3/4]", "no counter"] {
|
||
let re = FILE_COUNTER.captures(t.as_bytes()).and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap()));
|
||
assert_eq!(file_counter_total(t), re, "{t}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn poster_tags_as_in_the_typescript() {
|
||
let kill = "www.ggb.6x.to <darkness@earth.de>";
|
||
let a = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].r00");
|
||
let nfo = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (01/10) "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo" yEnc"#, "Cephalgy.-.Weisses.Fleisch.2003[[g.6x.to]].nfo");
|
||
let b = release_identity(kill, r#"[KiLL.dA.FuCkIn.EaStErRaBbIt]---- (03/10) "Shnarph.-.Lautlos.2008[[g.6x.to]].r00" yEnc"#, "Shnarph.-.Lautlos.2008[[g.6x.to]].r00");
|
||
assert_eq!(a.key, nfo.key);
|
||
assert_ne!(a.key, b.key);
|
||
assert_eq!(a.source, "stem");
|
||
let ftd1 = release_identity("p", r#"FTD769997 [01/23] - "Prison.Break.S04E02.720p.part01.rar" yEnc"#, "Prison.Break.S04E02.720p.part01.rar");
|
||
let ftd2 = release_identity("p", r#"FTD769997 [23/23] - "Prison.Break.S04E02.720p.part.par2" yEnc"#, "Prison.Break.S04E02.720p.part.par2");
|
||
let kere = |n: &str| release_identity("p", &format!("<kere.ws/illuminatenboard.org> - ID - 1855891249 - {n}"), n);
|
||
assert_ne!(kere("Along.The.Ridge.2006.Limited.PAL.DVDr-TWCiSO.rar").key, kere("Altera.QUARTUS.II.DSP.Builder.v7.2.SP1-SHooTERS.rar").key);
|
||
assert_eq!(ftd1.key, ftd2.key);
|
||
let scene = release_identity("p", r#"Some.Movie.2008.DVDRip.XviD-GRP [01/40] - "grp-sm.part01.rar" yEnc"#, "grp-sm.part01.rar");
|
||
assert_eq!(scene.name, "Some Movie 2008 DVDRip XviD GRP");
|
||
assert!(is_poster_tag("PowBeertje80 Posted", "Bravo Hits Vol.14 2CD -1996-", false));
|
||
assert!(!is_poster_tag("Audiofiles DIE ZEIT 35", "35_China_dl", false));
|
||
}
|
||
|
||
#[test]
|
||
fn sets_documents_and_audio_as_in_the_typescript() {
|
||
let id = |subject: &str, name: &str| release_identity("p", subject, name);
|
||
let a = id(r#"What about this? - File 027 of 132: "DSC_027.jpg" yEnc"#, "DSC_027.jpg");
|
||
let b = id(r#"What about this? - File 028 of 132: "DSC_028.jpg" yEnc"#, "DSC_028.jpg");
|
||
let other = id(r#"What about this? - File 001 of 90: "DSC_001.jpg" yEnc"#, "DSC_001.jpg");
|
||
assert_eq!(a.key, b.key);
|
||
assert_ne!(a.key, other.key);
|
||
assert_eq!(a.name, "What about this?");
|
||
let k1 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-76-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-76-lg.jpg");
|
||
let k2 = id(r#"Proud to be a poster "Kitchen Hottie 1 set233-77-lg.jpg" yEnc"#, "Kitchen Hottie 1 set233-77-lg.jpg");
|
||
let k3 = id(r#"Proud to be a poster "Office Chair set12-01-lg.jpg" yEnc"#, "Office Chair set12-01-lg.jpg");
|
||
assert_eq!(k1.key, k2.key);
|
||
assert_ne!(k1.key, k3.key);
|
||
for (subject, expected) in [
|
||
(r#"Minutemen (001/104) "x.cbr" - 76.88 MB - yEnc"#, "Minutemen"),
|
||
(r#"Babes P A R T 3 ("Charlotte 03 16.jpg" 0097-1574 156.87 kB-273.03 MB) yEnc"#, "Babes P A R T 3"),
|
||
(r#"*MaryKate* - OfficeChair [23of61] - "OfficeChair023.jpg" yEnc 187924 bytes"#, "*MaryKate* OfficeChair"),
|
||
(r#"Adventure Comics [1948-12] - "x.rar""#, "Adventure Comics [1948 12]"),
|
||
] {
|
||
assert_eq!(subject_remainder(subject), expected, "{subject}");
|
||
assert_eq!(subject_remainder_re(subject), expected, "{subject}");
|
||
}
|
||
let c1 = id(r#"cw-horror&mystery [10/28] "Vampirella Magazine 061.cbr" yEnc"#, "Vampirella Magazine 061.cbr");
|
||
let c2 = id(r#"cw-horror&mystery [11/28] "Mystery Tales 12.cbr" yEnc"#, "Mystery Tales 12.cbr");
|
||
assert_ne!(c1.key, c2.key);
|
||
assert_eq!(c2.name, "Mystery Tales 12");
|
||
let t1 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [04/59] - "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR" yEnc"#, "042 - 011 - Der unheimliche Bogenschütze.mp3 - by DeckaR");
|
||
let t2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [34/59] - "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR" yEnc"#, "072 - 027 - Die Teufelsuhr.MP3 - by DeckaR");
|
||
let par2 = id(r#"(Geisterjäger John Sinclair Hörspiele Vol. 2) [59/59] - "Hörspiele Vol. 2.par2" yEnc"#, "Hörspiele Vol. 2.par2");
|
||
assert_eq!(t1.key, t2.key);
|
||
assert_eq!(t1.key, par2.key);
|
||
// A cover without words around it joins its stem's archive.
|
||
assert_eq!(id(r#"[2/6] - "x.jpg" yEnc"#, "x.jpg").key, id(r#"[1/6] - "x.part01.rar" yEnc"#, "x.part01.rar").key);
|
||
}
|
||
|
||
#[test]
|
||
fn random_names() {
|
||
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||
assert!(looks_random("RmISakytFwXSCEzjVEpaKk"));
|
||
assert!(looks_random("51294P1CUKR2R35X"));
|
||
assert!(!looks_random("COD4MW2REMASTERED"));
|
||
assert!(!looks_random("BEATLES1964REMASTER"));
|
||
assert!(!looks_random("Some.Show.S01E01.1080p"));
|
||
assert!(looks_random("VIDEO_TS"));
|
||
}
|
||
|
||
#[test]
|
||
fn tools() {
|
||
assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu");
|
||
assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost");
|
||
assert_eq!(upload_tool("x@y"), "other");
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod numbered_tests {
|
||
use super::release_identity;
|
||
|
||
#[test]
|
||
fn numbered_items_of_a_set_stay_one_release() {
|
||
let poster = "Poster49118 <Poster49118@generic.mail>";
|
||
let a = release_identity(poster, r#"ai-engineer-complete [401/465] - "003 Defining Tools with JSON Schema.mp4" yEnc"#, "003 Defining Tools with JSON Schema.mp4");
|
||
let b = release_identity(poster, r#"ai-engineer-complete [422/465] - "006 ai-lesson-4-5-web-search-tools.zip" yEnc"#, "006 ai-lesson-4-5-web-search-tools.zip");
|
||
assert_eq!(a.key, b.key);
|
||
assert_eq!(a.name, "ai engineer complete");
|
||
let sc = |t: &str| release_identity("p", &format!("Spotlight - 83xx's [16/19] - \"{t}.zip\" yEnc"), &format!("{t}.zip"));
|
||
assert_eq!(sc("SC8367-12 - Wynette, Tammy - Singing My Song").key, sc("SC8367-13 - Jones, George - He Stopped Loving Her Today").key);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod image_set_tests {
|
||
use super::{release_identity, subject_remainder, subject_remainder_re};
|
||
|
||
#[test]
|
||
fn random_posters_are_no_part_of_the_key() {
|
||
use super::random_poster;
|
||
let ftd = "-Alight_Upon_the_Lake_(Live_at_the_Jazz_Showcase_Jun_1975)-WEB-2026-FTD";
|
||
let a = release_identity("N9vplIaavBCz@aCTVoAsk.JSE", &format!("[1/6] - \"Yusef_Lateef{ftd}.par2\" yEnc 1744"), &format!("Yusef_Lateef{ftd}.par2"));
|
||
let b = release_identity("FVWWHku0fGAg@QfaTz0g0.RVH", &format!("[3/6] - \"Yusef_Lateef{ftd}.part2.rar\" yEnc 104857600"), &format!("Yusef_Lateef{ftd}.part2.rar"));
|
||
assert_eq!(a.key, b.key);
|
||
assert!(random_poster("N9vplIaavBCz@aCTVoAsk.JSE"));
|
||
assert!(!random_poster("Captain Salvo <Capt@Salvo.com>"));
|
||
assert!(!random_poster("Poster49118 <Poster49118@generic.mail>"));
|
||
assert!(!random_poster("w.merklein@freenet.de (muffin)"));
|
||
}
|
||
|
||
#[test]
|
||
fn image_sets_without_suffixes() {
|
||
use super::image_set;
|
||
for (stem, want) in [
|
||
("stp250x001", "stp250x"),
|
||
("cutie-071-032", "cutie-071-"),
|
||
("katelg010_10", "katelg"),
|
||
("kate112lg035_35", "kate112lg"),
|
||
("!!hgrscans1097_2009", "!!hgrscans"),
|
||
("pj02kristina_24of40_1600", "pj02kristina_"),
|
||
("keira_s04_l keira_s04_0040_l", "keira_s04_l keira_s04_"),
|
||
("IMG_2676", "IMG_"),
|
||
("2877-041", "2877-"),
|
||
("051 - White leather kate054001_1_1", "051 - White leather kate"),
|
||
("108 - Schoolwall kate108lg029_28", "108 - Schoolwall kate108lg"),
|
||
("kurozo-2006-7-26-15-21", ""),
|
||
("bwp2_037_5", "bwp2_"),
|
||
("Set 008 10of63 new0464", "Set 008 "),
|
||
("202/430 princess-079-058", ""),
|
||
("( 098 /500) 002 148", ""),
|
||
("New Folder (17) sexypattycake_0903040445[1]", "New Folder ("),
|
||
("092_081_2000x1500", "092_"),
|
||
("KP0027_197_00", "KP"),
|
||
("01079-106-2048px", "01079-"),
|
||
("diana-050-052[1][1]", "diana-050-"),
|
||
("sss-70556-3474182-23-10825418-pc", ""),
|
||
("Alena p 26663_alena-p-2-8323_123_39lo", ""),
|
||
("she has sensual legs 172 _ 172", "she has sensual legs "),
|
||
("jamie.lee.curtis_the.tailor.of.panama_17_123_244lo", "jamie.lee.curtis_the.tailor.of.panama_"),
|
||
("JimAndJoey Kindly Husband-to-Be", ""),
|
||
] {
|
||
assert_eq!(image_set(stem), want, "{stem}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn image_sets_by_their_numbered_name() {
|
||
let silver = |n: &str| release_identity("p", &format!(r#"The excellent STIRILING SILVER - Always popular - "{n}" yEnc 320794 bytes"#), n);
|
||
assert_eq!(silver("stp250x001.jpg").key, silver("stp250x002.jpg").key);
|
||
assert_ne!(silver("stp250x001.jpg").key, silver("stp305x001.jpg").key);
|
||
let mint = |n: &str| release_identity("Cert", &format!(r#"I'm not a breath mint, but I am not all there either. [{n}" yEnc]"#), n);
|
||
assert_eq!(mint("JimAndJoey Kindly Husband-to-Be.JPG").key, mint("Oops OnlyOneMore.JPG").key);
|
||
}
|
||
|
||
#[test]
|
||
fn other_quoted_text_and_text_separators_name_the_release() {
|
||
let cc = |name: &str, n: &str| release_identity("p", &format!(r#">>> CREATIVE COMMONS NZB <<< "{name} " - File 1 of 9: "creative_commons_nzb_{n}.rar" yEnc"#), &format!("creative_commons_nzb_{n}.rar"));
|
||
assert_ne!(cc("0190 alarm 5.0", "0190_alarm_5.0_").key, cc("portable irfanview 4.22", "portable_irfanview_4.22").key);
|
||
let vlad = |dir: &str, total: u32, n: &str| release_identity("p", &format!(r#"Posting & deleting, read 0 file "E:\VladModels\y038_Irina\{dir}" Day 5/6 [01/{total}] - {n} yEnc"#), n);
|
||
assert_eq!(vlad("185", 77, "P8250001.JPG").source, "subject");
|
||
assert_ne!(vlad("185", 77, "P8250001.JPG").key, vlad("234", 77, "P5300085.JPG").key);
|
||
}
|
||
|
||
#[test]
|
||
fn hex_ids_dated_stems_and_season_packs() {
|
||
let nz = |id: &str, i: u32, n: &str| release_identity("nEwZ", &format!(r#"[PRiVATE] Rosewood.S02E08.SPANiSH.1080p.HDTV.x264-FCC [{id}] [newzNZB] [{i:02}/22] - "{n}" yEnc"#), n);
|
||
assert_eq!(nz("1f1793f68d5321", 1, "[05cbdb538a]_x.nfo").key, nz("65e6464bc03cad", 2, "[e8a4e2c182]_x.r03").key);
|
||
let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Coast To Coast Radio Show - "{n}" yEnc"#), n);
|
||
assert_eq!(show("09-30-2008_CTC.nfo").key, show("09-30-2008_CTC.r00").key);
|
||
assert_ne!(show("09-30-2008_CTC.r00").key, show("10-01-2008_CTC.r00").key);
|
||
let ep = |i: u32, n: &str| release_identity("p", &format!(r#"[{i:02}/36] "{n}""#), n);
|
||
assert_eq!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").key, ep(2, "Pokemon - S02E02 - Gotta Catch Ya Later.mkv").key);
|
||
assert_eq!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").name, "Pokemon S02");
|
||
assert_ne!(ep(1, "Pokemon - S02E01 - Pallet Party Panic.mkv").key, ep(1, "Pokemon - S03E01 - x.mkv").key);
|
||
}
|
||
|
||
#[test]
|
||
fn scene_index_files_go_with_the_release() {
|
||
let p = "SavageCobra <x@y.nhc>";
|
||
let a = release_identity(p, "00-laura_van_dam_-_capture_radio_113-cable-10-10-2026-talion.m3u", "00-laura_van_dam_-_capture_radio_113-cable-10-10-2026-talion.m3u");
|
||
let b = release_identity(p, "Laura_van_Dam_-_Capture_Radio_113-CABLE-10-10-2026-TALiON.vol000+001.par2", "Laura_van_Dam_-_Capture_Radio_113-CABLE-10-10-2026-TALiON.vol000+001.par2");
|
||
assert_eq!(a.key, b.key);
|
||
assert_eq!(b.name, "Laura_van_Dam_-_Capture_Radio_113-CABLE-10-10-2026-TALiON");
|
||
}
|
||
|
||
#[test]
|
||
fn split_rar_volumes_and_sizes_after_the_counter() {
|
||
assert_eq!(super::file_stem("rld-gtre.r13.002"), "rld-gtre");
|
||
for (n, stem) in [
|
||
("BD_NoTHing018.s41", "BD_NoTHing018"),
|
||
("x8ZRmX5N83mrokOwqJtoIGHTrqXQrgCyj.s00", "x8ZRmX5N83mrokOwqJtoIGHTrqXQrgCyj"),
|
||
("the.office.us.s01.rar", "the.office.us.s01"),
|
||
("Some.Show.S01", "Some.Show.S01"),
|
||
("Electric.Rain.Swift.3D.PS.v1.0.142.x64", "Electric.Rain.Swift.3D.PS.v1.0.142.x64"),
|
||
("App.x86", "App.x86"),
|
||
("Licence 700790900887990.z18", "Licence 700790900887990"),
|
||
("Licence 077887994400668.x12", "Licence 077887994400668"),
|
||
("Reply 987986177631730.x004", "Reply 987986177631730"),
|
||
("goh1Phoz5ahXu.par.vol006+08.par2", "goh1Phoz5ahXu"),
|
||
("brSWc0Q2KWblQpogOmgbn9.part.par2", "brSWc0Q2KWblQpogOmgbn9"),
|
||
("x.parts.par2", "x.parts"),
|
||
("Some.Movie.2010.x264", "Some.Movie.2010.x264"),
|
||
("chappelle1.part02.rev", "chappelle1"),
|
||
("chappelle1.rev", "chappelle1"),
|
||
("VwE3i90MVX9dqMFuDfzdS7Q6.part1.par2", "VwE3i90MVX9dqMFuDfzdS7Q6"),
|
||
("VwE3i90MVX9dqMFuDfzdS7Q6.part1.vol00+34.par2", "VwE3i90MVX9dqMFuDfzdS7Q6"),
|
||
("Movie.Part1.mkv", "Movie.Part1"),
|
||
("NZPECQUztWOponTpCratvvTPDBXBjvHIULOUPvSH.180.bc", "NZPECQUztWOponTpCratvvTPDBXBjvHIULOUPvSH"),
|
||
("x.12345.bc", "x.12345"),
|
||
] {
|
||
assert_eq!(super::file_stem(n), stem, "{n}");
|
||
assert_eq!(super::file_stem_re(n), stem, "{n}");
|
||
}
|
||
assert_eq!(super::size_after_name("(Hot MILF getting banged hard)[27/70] - 2407.jpg - 28", "2407.jpg"), "(Hot MILF getting banged hard)[27/70] - 2407.jpg");
|
||
assert_eq!(super::size_after_name(r#"Show [01/10] - "a.jpg" - 28"#, "a.jpg"), r#"Show [01/10] - "a.jpg" - 28"#);
|
||
assert_eq!(super::size_after_name("Show - Episode 05", "x.mkv"), "Show - Episode 05");
|
||
for (n, cut) in [
|
||
("lois 1258 Billy Joel - Extremes.part01.rar lois 1258 Billy Joel - Extremes ", "lois 1258 Billy Joel - Extremes.part01.rar"),
|
||
("GrooveCoverage.part26.rarGrooveCoveragebestof", "GrooveCoverage.part26.rar"),
|
||
("Goga Sekulic.part134.rar - Goga Sekulic - Diskografija (2000-2014).", "Goga Sekulic.part134.rar"),
|
||
("HASf293jLAIWUELvhaG3.part091.rar HASf293jLAIWUELvhaG3", "HASf293jLAIWUELvhaG3.part091.rar"),
|
||
("Movie.rar.Collection.mkv", "Movie.rar.Collection.mkv"),
|
||
("Angry.Birds.Rio.v1.2.2.cracked.READ.NFO-THETA.rar", "Angry.Birds.Rio.v1.2.2.cracked.READ.NFO-THETA.rar"),
|
||
("x.part01.rar", "x.part01.rar"),
|
||
] {
|
||
assert_eq!(super::run_on(n), cut, "{n}");
|
||
}
|
||
let lois = |i: &str, n: &str| super::release_identity("p", &format!(r#"(thnx to original poster) [{i}/92] - "{n}" yEnc"#), n);
|
||
assert_eq!(
|
||
lois("02", "lois 1258 Billy Joel - Extremes.part01.rar lois 1258 Billy Joel - Extremes ").key,
|
||
lois("01", "lois 1258 Billy Joel - Extremes.par2 lois 1258 Billy Joel - Extremes ").key
|
||
);
|
||
let milf = |i: &str, n: &str, kb: &str| super::release_identity("p", &format!("(Hot MILF getting banged hard)[{i}/70] - {n} - {kb}"), n);
|
||
assert_eq!(milf("27", "2407.jpg", "28").key, milf("28", "2408.jpg", "31").key);
|
||
assert_eq!(super::file_stem_re("rld-gtre.r13.002"), "rld-gtre");
|
||
let k = |i: u32, size: u32, n: &str| release_identity("Kingdn", &format!(r#"Buffy The Vampire Slayer s02e04 [{i}/32] - [{size}] "{n}" yEnc"#), n);
|
||
assert_eq!(k(26, 4929, "Buffy s02e04 xvid-tns.part24.rar").key, k(27, 400, "Buffy s02e04 xvid-tns.vol00+01.PAR2").key);
|
||
}
|
||
|
||
#[test]
|
||
fn totals_padded_wider_than_the_index_are_no_counters() {
|
||
let who = |i: u32, n: &str| release_identity("p", &format!(r#"Very Best Of The Who [{i:02}/{:03}] - "{n}" yEnc"#, i * 10), n);
|
||
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").key, who(4, "The Who - 02 - Anyway.mp3").key);
|
||
assert_eq!(who(3, "The Who - 01 - I Can't Explain.mp3").file_total, None);
|
||
assert_eq!(super::counter_total(r#"x [001/010] - "a.rar""#).as_deref(), Some("10"));
|
||
}
|
||
|
||
#[test]
|
||
fn dates_lead_no_numbered_items_and_repeated_names_go() {
|
||
let show = |n: &str| release_identity("#sterntuary <aa@aa.com>", &format!(r#"#sterntuary - Jeff Rense Radio Program - "{n}" yEnc"#), n);
|
||
assert_eq!(show("09-30-2008_BAK_JRS.nfo").key, show("09-30-2008_BAK_JRS.r00").key);
|
||
assert_ne!(show("09-30-2008_BAK_JRS.r00").key, show("10-01-2008_BAK_JRS.r00").key);
|
||
let nelet = |n: &str, i: u32| release_identity("p", &format!(r#"({n}) [{i:02}/58] - "{n}" yEnc"#), n);
|
||
assert_eq!(nelet("NELET11.par2", 1).key, nelet("NELET11.part01.rar", 2).key);
|
||
let wl = |n: &str| release_identity("p", &format!(r#"(01/26) "{n}" "Silicon Dream - Time Machine_BY_WaRLoRD```` yEnc"#), n);
|
||
assert_eq!(wl("01 Time Machine.mp3").key, wl("02 Jimmy Dean Loved Marilyn.mp3").key);
|
||
}
|
||
|
||
#[test]
|
||
fn host_and_sponsor_without_counter() {
|
||
let tn = |tag: &str, n: &str| release_identity("p", &format!(r#"(www.Thunder-News.org) >{tag}< <Sponsored by Secretusenet> - "{n}" yEnc"#), n);
|
||
assert_eq!(tn("XXX", "ps-itb10b.r00").key, tn("XXX", "ps-itb10b.par2").key);
|
||
assert_ne!(tn("XXX", "ps-itb10b.r00").key, tn("XXX", "swe6-legsa.r00").key);
|
||
assert_eq!(tn("XXX", "ps-itb10b.r00").source, "stem");
|
||
let named = tn("For.Rent.XXX.720p.Bluray.x264-DPXXXHD", "dp-for.rent.720p.x264.r00");
|
||
assert_eq!(named.source, "subject");
|
||
assert_eq!(named.key, tn("For.Rent.XXX.720p.Bluray.x264-DPXXXHD", "dp-for.rent.720p.x264.par2").key);
|
||
assert_eq!(named.name, ">For Rent XXX 720p Bluray x264 DPXXXHD<");
|
||
let counted = release_identity("p", r#"(www.Thunder-News.org) >XXX< [01/20] - "ps-itb10b.r00" yEnc"#, "ps-itb10b.r00");
|
||
assert_eq!(counted.source, "subject");
|
||
}
|
||
|
||
#[test]
|
||
fn hashed_stems_volume_numbers_and_scene_tracks() {
|
||
let u4e = |n: &str| release_identity("p", &format!(r#"Seit 1753 Tagen - usenet4ever.info - "{n}" yEnc"#), n);
|
||
assert_eq!(u4e("055631a1_b6c820bc_a856086c_85112a5-u4e.part01.rar").key, u4e("055631a1_b6c820bc_a856086c_85112a5-u4e.par2").key);
|
||
assert_ne!(u4e("055631a1_b6c820bc_a856086c_85112a5-u4e.part01.rar").key, u4e("4cc7c4c6_d3c2fecf_0bff51d6_d10df86-u4e.part01.rar").key);
|
||
let gb = |n: &str| release_identity("p", &format!(r#"German Board - [Usenet4ever.info] und [SecretUsenet.com] - "{n}" yEnc"#), n);
|
||
assert_eq!(gb("6c57319fbae505679737bd46a76193a1-u4e.part6.rar").key, gb("6c57319fbae505679737bd46a76193a1-u4e.par2").key);
|
||
assert_ne!(gb("6c57319fbae505679737bd46a76193a1-u4e.part6.rar").key, gb("6fc0ff192a355703b3e81557893013b1-u4e.part6.rar").key);
|
||
assert_ne!(gb("21.Jump.Street.2012.BDRip.AC3.German.XviD-POE.part01.rar").key, gb("Ice.Age.4.Voll.verschoben.2012.BDRip.AC3.German.XviD-POE.part01.rar").key);
|
||
assert_eq!(gb("21.Jump.Street.2012.BDRip.AC3.German.XviD-POE.part01.rar").key, gb("21.Jump.Street.2012.BDRip.AC3.German.XviD-POE.par2").key);
|
||
let mac = |i: &str, n: &str| release_identity("p", &format!(r#"[1598]-[FULL]-[#a.b.teevee@EFNet]-[ MacGyver.S01E03.DVDRip.XviD-MEDiEVAL ]-[{i}/38] - "{n}" yEnc"#), n);
|
||
assert_eq!(mac("04", "sample.macgy.s01e03-med.avi").key, mac("05", "macgy.s01e03-med.r00").key);
|
||
let pck = |i: &str, n: &str| release_identity("p", &format!(r#"(????) [{i}/2195] - "{n}" yEnc"#), n);
|
||
assert_eq!(pck("0659", "DN.VPL PCK.1508").key, pck("0660", "DN.VPL PCK.1509").key);
|
||
assert_eq!(pck("0659", "DN.VPL PCK.1508").name, "DN.VPL PCK.1508");
|
||
let movie = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/30] - "{n}" yEnc"#), n);
|
||
assert_ne!(movie("01", "Movie.2008").key, movie("02", "Movie.2009").key);
|
||
let track = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/14] - "{n}" yEnc"#), n);
|
||
assert_eq!(track("06", "02-repiet_-_dont_think_twice_(extended_mix).mp3").key, track("07", "03-repiet_-_other_song.mp3").key);
|
||
assert_eq!(track("06", "02-repiet_-_dont_think_twice_(extended_mix).mp3").name, "repiet");
|
||
assert_eq!(track("01", "01-2bm_mq-real_rap_(intro).mp3").name, "2bm_mq");
|
||
let meow = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/13] - "{n}" yEnc"#), n);
|
||
let tracks = meow("05", "01-ammar-turk_uit_de_kroeg.mp3").key;
|
||
for n in ["00-ammar-turk_uit_de_kroeg-single-web-nl-2022-cover.jpg", "00-ammar-turk_uit_de_kroeg-single-web-nl-2022.nfo", "00-ammar-turk_uit_de_kroeg-single-web-nl-2022.sfv"] {
|
||
assert_eq!(meow("01", n).key, tracks, "{n}");
|
||
}
|
||
assert_ne!(meow("01", "00-frank_van_etten-wie_heeft_jouw_lach_gestolen-single-web-nl-2022-cover.jpg").key, tracks);
|
||
let blaise = |i: &str, n: &str| release_identity("p", &format!(r#"[{i}/14] - "{n}" yEnc"#), n);
|
||
assert_eq!(blaise("07", "blaise_mc-in_hellstill_3-ep-web-2022-known.par2").key, blaise("03", "00-blaise_mc-in_hellstill_3-ep-web-2022.nfo").key);
|
||
assert_eq!(blaise("09", "blaise_mc-in_hellstill_3-ep-web-2022-known.vol000+01.par2").key, blaise("05", "01-blaise_mc-intro.mp3").key);
|
||
assert_eq!(blaise("01", "Some.Movie.2010.1080p.BluRay.x264-GRP.par2").name, "Some.Movie.2010.1080p.BluRay.x264-GRP");
|
||
assert_ne!(
|
||
blaise("01", "Ace_Of_Base-Wheel_Of_Fortune-(861_545-2)-CDM-FLAC-1992-REETKEVER.par2").key,
|
||
blaise("01", "Ace_Of_Base-Other-(861_546-2)-CDM-FLAC-1993-REETKEVER.par2").key
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn images_without_words_by_their_set() {
|
||
let id = |n: &str, c: &str| release_identity("p", &format!(r#"##[ "{n}" ({c})"#), n);
|
||
assert_eq!(id("diana-040-001.jpg", "01/68").key, id("diana-040-002.jpg", "02/68").key);
|
||
assert_ne!(id("diana-040-001.jpg", "01/68").key, id("diana-407-081.jpg", "81/82").key);
|
||
}
|
||
|
||
#[test]
|
||
fn unquoted_names_are_no_counters() {
|
||
let shy = |n: &str| release_identity("p", &format!("{n} hardcore SHYLA STYLEZ & SHY LOVE"), n);
|
||
assert_eq!(shy("shyla-shy-09-0195.jpg").key, shy("shyla-shy-09-0766.jpg").key);
|
||
assert_eq!(shy("shyla-shy-09-0195.jpg").file_total, None);
|
||
}
|
||
|
||
#[test]
|
||
fn dates_are_no_counters() {
|
||
use super::counter_total;
|
||
assert_eq!(counter_total("Avira AntiVir Rescue System 08-29-2011.rar"), None);
|
||
assert_eq!(counter_total("Android Apps and Games Pack 27-02-2011.rar"), None);
|
||
assert_eq!(counter_total("Supernova-clubbin (slamfm)-cable-18-08-2012-1king.rar"), None);
|
||
assert_eq!(counter_total(r#"Some.Album [03/12] - "x.mp3""#).as_deref(), Some("12"));
|
||
assert_eq!(counter_total(r#"File 07390 of 19411 "x.epub""#).as_deref(), Some("19411"));
|
||
}
|
||
|
||
#[test]
|
||
fn more_splits_from_the_reindex() {
|
||
use super::random_poster;
|
||
let makz = "Makz@BlueRadiance.uni (makz)";
|
||
// The name as unquoted_name gives it (spaces kept), the subject collapsed and without its counter.
|
||
let a = release_identity(makz, r"G:\posting\thegoodnudes\lara_bath 15of19 larabath_2.5.jpg", r"G:\posting\thegoodnudes\lara_bath 15of19 larabath_2.5.jpg");
|
||
let b = release_identity(makz, r"G:\posting\thegoodnudes\lara_bath 16of19 larabath_2.6.jpg", r"G:\posting\thegoodnudes\lara_bath 16of19 larabath_2.6.jpg");
|
||
assert_eq!(a.key, b.key);
|
||
let mara = "Mara <guest@cox.net>";
|
||
let a = release_identity(mara, r#"RP - File 268 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3""#, "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.16-22.mp3");
|
||
let b = release_identity(mara, r#"RP - File 269 of 295 - yEnc "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3""#, "Elizabeth Peters - (Peabody 09) Seeing A Large Cat - D11.17-22.mp3");
|
||
assert_eq!(a.key, b.key);
|
||
let mb = "misterb@sure.com (MisterB)";
|
||
let a = release_identity(mb, r#"Herb Ernst@256 - NMR -[07/17] "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3" - 8123 yEnc"#, "Herb Ernst - Dreamflight II - 05 - The Light of Forever.mp3");
|
||
let b = release_identity(mb, r#"Herb Ernst@256 - NMR -[08/17] "Herb Ernst - Dreamflight II - 06 - Other.mp3" - 9001 yEnc"#, "Herb Ernst - Dreamflight II - 06 - Other.mp3");
|
||
assert_eq!(a.key, b.key);
|
||
let destiny = |set: u32, i: u32| {
|
||
let name = format!("destiny-{set:03}-{i:03}.jpg");
|
||
release_identity("p", &format!(r#"Newstar Destiny I set {set} [{i} of 60] "{name}" yEnc"#), &name)
|
||
};
|
||
assert_eq!(destiny(1, 1).key, destiny(1, 2).key);
|
||
assert_ne!(destiny(1, 1).key, destiny(63, 34).key);
|
||
assert_eq!(destiny(63, 34).name, "Newstar Destiny I set 63");
|
||
for s in [r#"x - "a.mp3" - 8123 yEnc"#, "a8123 yEnc", "-8123 yEnc ", "yEnc", r#"set 1 [1 of 60] "x.jpg" yEnc"#, "Album 12 yEnc", "Album 1234 yEnc"] {
|
||
assert_eq!(super::subject_remainder(s), super::subject_remainder_re(s), "{s}");
|
||
}
|
||
assert!(random_poster("SOFZRw1-OTEd@EEFUbh.srG"));
|
||
assert!(random_poster("vMgGYlZB9ZcTNk8WxYFGV_OOW@x6YR.abc"));
|
||
assert!(!random_poster("w.merklein@freenet.de (muffin)"));
|
||
}
|
||
|
||
#[test]
|
||
fn sizes_in_brackets_and_one_word_sets() {
|
||
for s in [r#"Aletta Ocean - "100628-035.jpg" yEnc [347K]"#, r#"Corie - "2877-041.jpg" yEnc [290K]"#, "x - !!!blob2010-102katie009.jpg 336573 byt"] {
|
||
assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}");
|
||
}
|
||
assert_eq!(subject_remainder(r#"Aletta Ocean - "100628-035.jpg" yEnc [347K]"#), "Aletta Ocean");
|
||
let a = release_identity("p", r#"Corie - "2877-041.jpg" yEnc [290K]"#, "2877-041.jpg");
|
||
let b = release_identity("p", r#"Corie - "2877-042.jpg" yEnc [301K]"#, "2877-042.jpg");
|
||
assert_eq!(a.key, b.key);
|
||
assert_eq!(a.name, "Corie");
|
||
}
|
||
}
|
||
}
|