Daan MeijerandClaude Opus 5.5
b1c2db5984
Each with its regex/chrono version as the reference in tests; check.ts still finds all three captures identical. Boneless 2026 ~500k headers/s, ath 2013 ~650k. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
740 lines
28 KiB
Rust
740 lines
28 KiB
Rust
//! lib/release (identity, random), lib/files/volumes and lib/rules/upload_tools.
|
||
//! Regexes run on bytes without Unicode (`(?-u)`): JS regexes without the `u`
|
||
//! flag match ASCII classes, `\d` is [0-9] and `\b` is an ASCII boundary. JS
|
||
//! `\s` (which includes Unicode spaces) is spelled out as SP.
|
||
|
||
use crate::js::{self, len16, sha1_hex};
|
||
use md5::{Digest, Md5};
|
||
use regex::bytes::Regex;
|
||
use std::sync::LazyLock;
|
||
|
||
/// JS `\s` as a byte pattern.
|
||
const SP: &str = r"(?:[\t\n\x0B\x0C\r ]|\xC2\xA0|\xE1\x9A\x80|\xE2\x80[\x80-\x8A\xA8\xA9\xAF]|\xE2\x81\x9F|\xE3\x80\x80|\xEF\xBB\xBF)";
|
||
|
||
fn re(pattern: &str) -> Regex {
|
||
Regex::new(&format!("(?-u){}", pattern.replace("SP", SP))).unwrap()
|
||
}
|
||
|
||
/// `s.replace(regex, with)` (first match) or `s.replace(/…/g, with)` (all).
|
||
fn replace(re: &Regex, s: &str, with: &str, all: bool) -> String {
|
||
let out = if all { re.replace_all(s.as_bytes(), with.as_bytes()) } else { re.replacen(s.as_bytes(), 1, with.as_bytes()) };
|
||
// The patterns match whole characters (ASCII, or delimited by ASCII), so this stays UTF-8.
|
||
String::from_utf8(out.into_owned()).unwrap()
|
||
}
|
||
|
||
fn test(re: &Regex, s: &str) -> bool {
|
||
re.is_match(s.as_bytes())
|
||
}
|
||
|
||
static SUFFIXES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||
vec![
|
||
(re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"), ""),
|
||
(re(r"(?i)\.par2$"), ""),
|
||
(re(r"(?i)\.part[0-9]+\.rar$"), ""),
|
||
(re(r"(?i)\.r[0-9]{2,3}$"), ""),
|
||
(re(r"(?i)\.rar$"), ""),
|
||
(re(r"(?i)\.7z(\.[0-9]{3})?$"), ""),
|
||
(re(r"(?i)\.zip(\.[0-9]{3})?$"), ""),
|
||
// JS: (?<=\.(7z|…))\.\d{3}$ — no lookbehind here, so the extension is matched and put back.
|
||
(re(r"(?i)(\.(?:7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.[0-9]{3}$"), "$1"),
|
||
(re(r"(?i)\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$"), ""),
|
||
(re(r"(?i)\.tar$"), ""),
|
||
(re(r"(?i)\.(zst|gz|xz|bz2)$"), ""),
|
||
(re(r"(?i)\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$"), ""),
|
||
]
|
||
});
|
||
|
||
/// fileStem with the regexes, as in the TypeScript (the reference for file_stem's tests).
|
||
#[allow(dead_code)]
|
||
pub fn file_stem_re(name: &str) -> String {
|
||
let mut stem = js::trim(name).to_string();
|
||
let mut changed = true;
|
||
while changed {
|
||
changed = false;
|
||
for (suffix, with) in SUFFIXES.iter() {
|
||
if !suffix.is_match(stem.as_bytes()) {
|
||
continue;
|
||
}
|
||
let next = replace(suffix, &stem, with, false);
|
||
if next != stem && !next.is_empty() {
|
||
stem = next;
|
||
changed = true;
|
||
}
|
||
}
|
||
}
|
||
stem
|
||
}
|
||
|
||
static QUOTED: LazyLock<Regex> = LazyLock::new(|| re(r#""[^"]*""#));
|
||
static COUNTERS: LazyLock<Regex> = LazyLock::new(|| re(r"\[[0-9]+/[0-9]+\]|\([0-9]+/[0-9]+\)"));
|
||
static YENC_END: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b(SP+[0-9]+)?SP*$"));
|
||
static YENC: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\byEnc\b"));
|
||
static SEPARATORS: LazyLock<Regex> = LazyLock::new(|| re(r"(?:SP|[-.:|]|\xE2\x80\x93)+"));
|
||
|
||
/// subjectRemainder with the regexes, as in the TypeScript (the reference for subject_remainder's tests).
|
||
#[allow(dead_code)]
|
||
pub fn subject_remainder_re(subject: &str) -> String {
|
||
let s = replace("ED, subject, " ", true);
|
||
let s = replace(&COUNTERS, &s, " ", true);
|
||
let s = replace(&YENC_END, &s, " ", false);
|
||
let s = replace(&YENC, &s, " ", true);
|
||
let s = replace(&SEPARATORS, &s, " ", true);
|
||
js::trim(&s).to_string()
|
||
}
|
||
|
||
fn eq_ci(a: &[u8], b: &[u8]) -> bool {
|
||
a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_ascii_lowercase() == *y)
|
||
}
|
||
|
||
/// Ends in `suffix` (lowercase), ASCII case-insensitively.
|
||
fn ends_ci(s: &[u8], suffix: &[u8]) -> bool {
|
||
s.len() >= suffix.len() && eq_ci(&s[s.len() - suffix.len()..], suffix)
|
||
}
|
||
|
||
/// ASCII digits ending at `end`.
|
||
fn digits_back(s: &[u8], end: usize) -> usize {
|
||
s[..end].iter().rev().take_while(|c| c.is_ascii_digit()).count()
|
||
}
|
||
|
||
/// `.` + one of `exts` at the end: the length without it.
|
||
fn strip_ext(s: &[u8], exts: &[&[u8]]) -> Option<usize> {
|
||
for ext in exts {
|
||
let n = ext.len() + 1;
|
||
if s.len() >= n && s[s.len() - n] == b'.' && ends_ci(s, ext) {
|
||
return Some(s.len() - n);
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// `\.\d{3}$`: the length without it.
|
||
fn strip_three_digits(s: &[u8]) -> Option<usize> {
|
||
let n = s.len();
|
||
(n >= 4 && s[n - 4] == b'.' && s[n - 3..].iter().all(u8::is_ascii_digit)).then(|| n - 4)
|
||
}
|
||
|
||
const SPLIT_EXTS: [&[u8]; 14] = [b"7z", b"zip", b"rar", b"tar", b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"m2ts", b"iso", b"img", b"bin", b"vob"];
|
||
const SIDE_EXTS: [&[u8]; 19] = [b"nfo", b"sfv", b"nzb", b"srr", b"srs", b"txt", b"jpg", b"png", b"md5", b"sha1", b"srt", b"sub", b"idx", b"ass", b"ssa", b"m3u", b"m3u8", b"cue", b"log"];
|
||
const PACK_EXTS: [&[u8]; 4] = [b"zst", b"gz", b"xz", b"bz2"];
|
||
const MEDIA_EXTS: [&[u8]; 12] = [b"mkv", b"mp4", b"avi", b"m4v", b"ts", b"iso", b"img", b"bin", b"flac", b"mp3", b"epub", b"pdf"];
|
||
|
||
/// SUFFIXES[rule] once: the stem's new length when it matches.
|
||
fn strip_suffix_rule(s: &[u8], rule: usize) -> Option<usize> {
|
||
let n = s.len();
|
||
match rule {
|
||
// \.vol\d+[+-]\d+\.par2$
|
||
0 => {
|
||
if !ends_ci(s, b".par2") {
|
||
return None;
|
||
}
|
||
let d2 = digits_back(s, n - 5);
|
||
let sign = (n - 5).checked_sub(d2 + 1).filter(|_| d2 > 0)?;
|
||
if !matches!(s[sign], b'+' | b'-') {
|
||
return None;
|
||
}
|
||
let d1 = digits_back(s, sign);
|
||
let v = sign.checked_sub(d1 + 4).filter(|_| d1 > 0)?;
|
||
eq_ci(&s[v..v + 4], b".vol").then_some(v)
|
||
}
|
||
1 => ends_ci(s, b".par2").then(|| n - 5),
|
||
// \.part\d+\.rar$
|
||
2 => {
|
||
if !ends_ci(s, b".rar") {
|
||
return None;
|
||
}
|
||
let d = digits_back(s, n - 4);
|
||
let p = (n - 4).checked_sub(d + 5).filter(|_| d > 0)?;
|
||
eq_ci(&s[p..p + 5], b".part").then_some(p)
|
||
}
|
||
// \.r\d{2,3}$
|
||
3 => {
|
||
let d = digits_back(s, n);
|
||
let p = n.checked_sub(d + 2).filter(|_| (2..=3).contains(&d))?;
|
||
eq_ci(&s[p..p + 2], b".r").then_some(p)
|
||
}
|
||
4 => ends_ci(s, b".rar").then(|| n - 4),
|
||
// \.7z(\.\d{3})?$ and \.zip(\.\d{3})?$
|
||
5 | 6 => {
|
||
let ext: &[u8] = if rule == 5 { b".7z" } else { b".zip" };
|
||
if let Some(m) = strip_three_digits(s) {
|
||
if ends_ci(&s[..m], ext) {
|
||
return Some(m - ext.len());
|
||
}
|
||
}
|
||
ends_ci(s, ext).then(|| n - ext.len())
|
||
}
|
||
// (?<=\.(7z|…))\.\d{3}$: only the volume number goes.
|
||
7 => {
|
||
let m = strip_three_digits(s)?;
|
||
strip_ext(&s[..m], &SPLIT_EXTS).map(|_| m)
|
||
}
|
||
8 => strip_ext(s, &SIDE_EXTS),
|
||
9 => strip_ext(s, &[b"tar"]),
|
||
10 => strip_ext(s, &PACK_EXTS),
|
||
11 => strip_ext(s, &MEDIA_EXTS),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
/// fileStem: per-file suffixes stripped, repeatedly. The SUFFIXES regexes by hand: with them, a
|
||
/// file's description was most of the indexing time.
|
||
pub fn file_stem(name: &str) -> String {
|
||
let stem = js::trim(name);
|
||
let b = stem.as_bytes();
|
||
let mut len = b.len();
|
||
let mut changed = true;
|
||
while changed {
|
||
changed = false;
|
||
for rule in 0..12 {
|
||
if let Some(next) = strip_suffix_rule(&b[..len], rule) {
|
||
if next > 0 && next < len {
|
||
len = next;
|
||
changed = true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// The suffixes are ASCII: `len` is a character boundary.
|
||
stem[..len].to_string()
|
||
}
|
||
|
||
fn is_word(c: u8) -> bool {
|
||
c.is_ascii_alphanumeric() || c == b'_'
|
||
}
|
||
|
||
/// "yEnc" (any case) at `i` with a word boundary (ASCII, as in JS) on both sides.
|
||
fn yenc_at(b: &[u8], i: usize) -> bool {
|
||
i + 4 <= b.len() && eq_ci(&b[i..i + 4], b"yenc") && (i == 0 || !is_word(b[i - 1])) && (i + 4 == b.len() || !is_word(b[i + 4]))
|
||
}
|
||
|
||
/// `(\s+\d+)?\s*$`
|
||
fn yenc_tail(rest: &str) -> bool {
|
||
let after_spaces = rest.trim_start_matches(js::is_space);
|
||
if after_spaces.chars().all(js::is_space) {
|
||
return true;
|
||
}
|
||
if after_spaces.len() == rest.len() {
|
||
return false;
|
||
}
|
||
let after_digits = after_spaces.trim_start_matches(|c: char| c.is_ascii_digit());
|
||
after_digits.len() < after_spaces.len() && after_digits.chars().all(js::is_space)
|
||
}
|
||
|
||
/// subjectRemainder by hand: the same five replacements and trim.
|
||
pub fn subject_remainder(subject: &str) -> String {
|
||
let b = subject.as_bytes();
|
||
// "[^"]*" → ' '
|
||
let mut s1: Vec<u8> = Vec::with_capacity(b.len());
|
||
let mut i = 0;
|
||
while i < b.len() {
|
||
if b[i] == b'"' {
|
||
if let Some(j) = b[i + 1..].iter().position(|&c| c == b'"') {
|
||
s1.push(b' ');
|
||
i += j + 2;
|
||
continue;
|
||
}
|
||
}
|
||
s1.push(b[i]);
|
||
i += 1;
|
||
}
|
||
// \[\d+/\d+\] and \(\d+/\d+\) → ' '
|
||
let mut s2: Vec<u8> = Vec::with_capacity(s1.len());
|
||
let mut i = 0;
|
||
while i < s1.len() {
|
||
let close = match s1[i] {
|
||
b'[' => Some(b']'),
|
||
b'(' => Some(b')'),
|
||
_ => None,
|
||
};
|
||
if let Some(close) = close {
|
||
let a = s1[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let slash = i + 1 + a;
|
||
if a > 0 && s1.get(slash) == Some(&b'/') {
|
||
let d = s1[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if d > 0 && s1.get(slash + 1 + d) == Some(&close) {
|
||
s2.push(b' ');
|
||
i = slash + 2 + d;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
s2.push(s1[i]);
|
||
i += 1;
|
||
}
|
||
// Only ASCII was replaced, between whole characters.
|
||
let mut s3 = String::from_utf8(s2).unwrap();
|
||
// \byEnc\b(\s+\d+)?\s*$ → ' ' (the first place where it fits)
|
||
let b3 = s3.as_bytes();
|
||
if let Some(at) = (0..b3.len()).find(|&i| yenc_at(b3, i) && yenc_tail(&s3[i + 4..])) {
|
||
s3.truncate(at);
|
||
s3.push(' ');
|
||
}
|
||
// \byEnc\b → ' ', then runs of \s, -, –, ., :, | → ' '
|
||
let b3 = s3.as_bytes();
|
||
let mut s4 = String::with_capacity(s3.len());
|
||
let mut separator = false;
|
||
let mut i = 0;
|
||
while i < b3.len() {
|
||
if yenc_at(b3, i) {
|
||
// Its " " starts or joins a separator run.
|
||
if !separator {
|
||
s4.push(' ');
|
||
separator = true;
|
||
}
|
||
i += 4;
|
||
continue;
|
||
}
|
||
let c = s3[i..].chars().next().unwrap();
|
||
if js::is_space(c) || matches!(c, '-' | '.' | ':' | '|' | '\u{2013}') {
|
||
if !separator {
|
||
s4.push(' ');
|
||
separator = true;
|
||
}
|
||
} else {
|
||
s4.push(c);
|
||
separator = false;
|
||
}
|
||
i += c.len_utf8();
|
||
}
|
||
js::trim(&s4).to_string()
|
||
}
|
||
|
||
fn is_random_token(token: &str) -> bool {
|
||
let t = token.strip_prefix('[').unwrap_or(token);
|
||
let t = t.strip_suffix(']').unwrap_or(t);
|
||
if len16(t) < 6 {
|
||
return false;
|
||
}
|
||
let digit = t.bytes().any(|b| b.is_ascii_digit());
|
||
if digit && t.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||
return true;
|
||
}
|
||
digit && t.bytes().any(|b| b.is_ascii_lowercase()) && t.bytes().any(|b| b.is_ascii_uppercase()) && !t.bytes().any(|b| matches!(b, b'.' | b'_' | b'-'))
|
||
}
|
||
|
||
pub fn is_post_token(remainder: &str) -> bool {
|
||
let compact: String = remainder.chars().filter(|&c| !js::is_space(c) && !matches!(c, '[' | ']' | '(' | ')' | '_' | '-')).collect();
|
||
let n = len16(&compact);
|
||
(6..=40).contains(&n) && compact.bytes().any(|b| b.is_ascii_digit()) && compact.bytes().all(|b| b.is_ascii_alphanumeric())
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
static NAME_LIKE: LazyLock<Regex> = LazyLock::new(|| re(r"[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}"));
|
||
|
||
pub fn names_release(remainder: &str) -> bool {
|
||
let words: Vec<&str> = js::split_spaces(remainder).into_iter().filter(|w| !w.is_empty() && !is_random_token(w)).collect();
|
||
name_like(words.join(" ").as_bytes())
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct Release {
|
||
pub key: String,
|
||
pub name: String,
|
||
pub source: &'static str,
|
||
/// The [i/M] total as parsed (0 is falsy in the TypeScript).
|
||
pub file_total: Option<i64>,
|
||
}
|
||
|
||
impl Release {
|
||
pub fn file_total_truthy(&self) -> Option<i64> {
|
||
self.file_total.filter(|&n| n != 0)
|
||
}
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
static FILE_COUNTER: LazyLock<Regex> = LazyLock::new(|| re(r"\[([0-9]+)/([0-9]+)\]"));
|
||
|
||
fn hash(parts: &[&str]) -> String {
|
||
sha1_hex(&parts.join("\n"))
|
||
}
|
||
|
||
/// releaseIdentity for a named (not weak) file: never null.
|
||
pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||
let file_total = file_counter_total(subject);
|
||
let stem = file_stem(name);
|
||
let remainder = subject_remainder(subject);
|
||
if names_release(&remainder) {
|
||
return Release { key: hash(&["subject", poster, &remainder]), name: remainder, source: "subject", file_total };
|
||
}
|
||
if let Some(total) = file_total.filter(|&n| n != 0) {
|
||
if is_post_token(&remainder) {
|
||
return Release { key: hash(&["post", poster, &remainder, &total.to_string()]), name: remainder, source: "post", file_total };
|
||
}
|
||
}
|
||
Release { key: hash(&["stem", poster, &stem]), name: stem, source: "stem", file_total }
|
||
}
|
||
|
||
static GENERIC: LazyLock<Regex> = LazyLock::new(|| {
|
||
re(r"(?i)^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?[0-9]{1,2})$")
|
||
});
|
||
|
||
fn is_generic_name(name: &str) -> bool {
|
||
test(&GENERIC, js::trim(name))
|
||
}
|
||
|
||
/// looksRandom.
|
||
pub fn looks_random(name: &str) -> bool {
|
||
// name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||
let name = match name.rfind('.') {
|
||
Some(i) if (1..=4).contains(&(name.len() - i - 1)) && name[i + 1..].bytes().all(|b| b.is_ascii_alphanumeric()) => &name[..i],
|
||
_ => name,
|
||
};
|
||
if is_generic_name(name) {
|
||
return true;
|
||
}
|
||
let b = name.as_bytes();
|
||
let hex = |s: &[u8]| s.iter().all(|c| c.is_ascii_hexdigit());
|
||
let digits = |s: &[u8]| s.iter().all(|c| c.is_ascii_digit());
|
||
if b.len() >= 16 && hex(b) {
|
||
return true;
|
||
}
|
||
if let Some(dot) = name.find('.') {
|
||
let (h, d) = (&b[..dot], &b[dot + 1..]);
|
||
if h.len() >= 16 && hex(h) && d.len() >= 9 && digits(d) {
|
||
return true;
|
||
}
|
||
}
|
||
if b.len() == 36 && [8, 13, 18, 23].iter().all(|&i| b[i] == b'-') && b.iter().enumerate().all(|(i, c)| [8, 13, 18, 23].contains(&i) || c.is_ascii_hexdigit()) {
|
||
return true;
|
||
}
|
||
{
|
||
let core = name.strip_suffix(" n").unwrap_or(name).as_bytes();
|
||
if core.len() >= 6 && digits(core) {
|
||
return true;
|
||
}
|
||
}
|
||
let count = |f: fn(&u8) -> bool| b.iter().filter(|c| f(c)).count();
|
||
let n_digit = count(u8::is_ascii_digit);
|
||
let n_lower = count(u8::is_ascii_lowercase);
|
||
let n_upper = count(u8::is_ascii_uppercase);
|
||
let alnum = b.iter().all(|c| c.is_ascii_alphanumeric());
|
||
if b.len() >= 12 && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()) && n_digit >= 2 && n_lower >= 4 {
|
||
return true;
|
||
}
|
||
if (8..=15).contains(&b.len()) && alnum && n_digit > 0 && n_lower > 0 && b[1..].iter().filter(|c| c.is_ascii_uppercase()).count() >= 3 {
|
||
return true;
|
||
}
|
||
if !(b.len() >= 16 && alnum) || n_lower == 0 || n_upper == 0 {
|
||
return false;
|
||
}
|
||
n_digit > 0 || n_upper >= 6
|
||
}
|
||
|
||
pub fn is_random_file(name: &str) -> bool {
|
||
looks_random(&file_stem(name))
|
||
}
|
||
|
||
static SEVEN_SPLIT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$"));
|
||
static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
|
||
static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
|
||
static RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.rar$"));
|
||
static PART: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part([0-9]+)\.rar$"));
|
||
static BIN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.bin$"));
|
||
static OLD_RAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.r[0-9]{2,3}$"));
|
||
static PAR2_VOL: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.vol[0-9]+[+-][0-9]+\.par2$"));
|
||
pub static PAR2: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.par2$"));
|
||
|
||
pub fn is_par2(name: &str) -> bool {
|
||
test(&PAR2, name)
|
||
}
|
||
|
||
pub fn seven_zip_volume(name: &str) -> Option<i64> {
|
||
if let Some(c) = SEVEN_SPLIT.captures(name.as_bytes()) {
|
||
return js::parse_int(std::str::from_utf8(&c[1]).unwrap());
|
||
}
|
||
if test(&SEVEN, name) {
|
||
Some(0)
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
|
||
pub fn is_archive_start(name: &str) -> bool {
|
||
if test(&BIN, name) {
|
||
return true;
|
||
}
|
||
if matches!(seven_zip_volume(name), Some(0) | Some(1)) {
|
||
return true;
|
||
}
|
||
test(&PART1, name) || (test(&RAR, name) && !test(&PART, name))
|
||
}
|
||
|
||
pub fn is_later_volume(name: &str) -> bool {
|
||
if let Some(c) = PART.captures(name.as_bytes()) {
|
||
return js::parse_int(std::str::from_utf8(&c[1]).unwrap()).unwrap_or(0) > 1;
|
||
}
|
||
test(&OLD_RAR, name) || test(&PAR2_VOL, name)
|
||
}
|
||
|
||
static TOOLS: LazyLock<Vec<(&'static str, Regex)>> = LazyLock::new(|| {
|
||
vec![
|
||
("nyuu", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@nyuu$")),
|
||
("nyuu-host", re(r"^(?:[A-Z][a-z]){12}-[0-9]{13}@[^\n\r]+$")),
|
||
("ngPost", re(r"^[0-9a-f]{32}@ngPost$")),
|
||
("yuun", re(r"^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$")),
|
||
("hex32", re(r"^[0-9a-f]{32}@[A-Za-z0-9.-]+$")),
|
||
("JBinUp", re(r"@JBin(?:Up|Down)\.local$")),
|
||
("partNofM", re(r"(?i)^part[0-9]+of[0-9]+\.")),
|
||
("ts-dollar", re(r"^[0-9]{13}\$[0-9]+\$[0-9]+@localhost$")),
|
||
("random", re(r"^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$")),
|
||
]
|
||
});
|
||
|
||
/// uploadTool with the regexes, as in the TypeScript (the reference for upload_tool's test).
|
||
#[allow(dead_code)]
|
||
pub fn upload_tool_re(message_id: &str) -> &'static str {
|
||
if message_id.is_empty() {
|
||
return "other";
|
||
}
|
||
for (tool, pattern) in TOOLS.iter() {
|
||
if test(pattern, message_id) {
|
||
return tool;
|
||
}
|
||
}
|
||
"other"
|
||
}
|
||
|
||
fn all(s: &[u8], f: impl Fn(u8) -> bool) -> bool {
|
||
s.iter().all(|&c| f(c))
|
||
}
|
||
|
||
fn lower_hex(c: u8) -> bool {
|
||
c.is_ascii_digit() || (b'a'..=b'f').contains(&c)
|
||
}
|
||
|
||
/// `(?:[A-Z][a-z]){12}-\d{13}@`: nyuu's prefix (38 bytes, then '@').
|
||
fn nyuu_prefix(b: &[u8]) -> bool {
|
||
b.len() > 38 && (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[24] == b'-' && all(&b[25..38], |c| c.is_ascii_digit()) && b[38] == b'@'
|
||
}
|
||
|
||
/// The posting tool of a Message-ID (TOOLS, in order, by hand: they ran for every per-article file).
|
||
pub fn upload_tool(message_id: &str) -> &'static str {
|
||
let b = message_id.as_bytes();
|
||
if b.is_empty() {
|
||
return "other";
|
||
}
|
||
if nyuu_prefix(b) && b.len() > 39 {
|
||
// nyuu: "@nyuu" exactly; nyuu-host: any host (`.+`: no line terminators).
|
||
if &b[39..] == b"nyuu" {
|
||
return "nyuu";
|
||
}
|
||
if !b[39..].iter().any(|&c| c == b'\n' || c == b'\r') && !message_id[39..].contains(['\u{2028}', '\u{2029}']) {
|
||
return "nyuu-host";
|
||
}
|
||
}
|
||
let hex32 = b.len() > 33 && all(&b[..32], lower_hex) && b[32] == b'@';
|
||
if hex32 {
|
||
let host = &b[33..];
|
||
if host == b"ngPost" {
|
||
return "ngPost";
|
||
}
|
||
if host.len() == 37 && all(&host[..32], lower_hex) && &host[32..] == b".yuun" {
|
||
return "yuun";
|
||
}
|
||
if all(host, |c| c.is_ascii_alphanumeric() || c == b'.' || c == b'-') {
|
||
return "hex32";
|
||
}
|
||
}
|
||
if message_id.ends_with("@JBinUp.local") || message_id.ends_with("@JBinDown.local") {
|
||
return "JBinUp";
|
||
}
|
||
// (?i)^part\d+of\d+\.
|
||
if b.len() > 4 && eq_ci(&b[..4], b"part") {
|
||
let d1 = b[4..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let of = 4 + d1;
|
||
if d1 > 0 && b.len() > of + 2 && eq_ci(&b[of..of + 2], b"of") {
|
||
let d2 = b[of + 2..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if d2 > 0 && b.get(of + 2 + d2) == Some(&b'.') {
|
||
return "partNofM";
|
||
}
|
||
}
|
||
}
|
||
// ^\d{13}\$\d+\$\d+@localhost$
|
||
if let Some(rest) = message_id.strip_suffix("@localhost") {
|
||
let r = rest.as_bytes();
|
||
let parts: Vec<&[u8]> = r.split(|&c| c == b'$').collect();
|
||
if parts.len() == 3 && parts[0].len() == 13 && parts.iter().all(|p| !p.is_empty() && all(p, |c| c.is_ascii_digit())) {
|
||
return "ts-dollar";
|
||
}
|
||
}
|
||
// ^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$
|
||
if let Some(at) = b.iter().position(|&c| c == b'@') {
|
||
let (local, host) = (&b[..at], &b[at + 1..]);
|
||
if (10..=40).contains(&local.len()) && all(local, |c| c.is_ascii_alphanumeric()) {
|
||
if let Some(dot) = host.iter().position(|&c| c == b'.') {
|
||
let (name, tld) = (&host[..dot], &host[dot + 1..]);
|
||
if (3..=12).contains(&name.len()) && (2..=6).contains(&tld.len()) && all(name, |c| c.is_ascii_alphanumeric()) && all(tld, |c| c.is_ascii_alphanumeric()) {
|
||
return "random";
|
||
}
|
||
}
|
||
}
|
||
}
|
||
"other"
|
||
}
|
||
|
||
/// [A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,} anywhere.
|
||
fn name_like(b: &[u8]) -> bool {
|
||
let sep = |c: u8| matches!(c, b'.' | b'_' | b' ' | b'-');
|
||
(2..b.len()).any(|k| {
|
||
if !(sep(b[k]) && b[k - 1].is_ascii_alphabetic() && b[k - 2].is_ascii_alphabetic()) {
|
||
return false;
|
||
}
|
||
let m = k + b[k..].iter().take_while(|&&c| sep(c)).count();
|
||
m + 2 <= b.len() && b[m].is_ascii_alphanumeric() && b[m + 1].is_ascii_alphanumeric()
|
||
})
|
||
}
|
||
|
||
/// The second number of the first \[(\d+)/(\d+)\].
|
||
fn file_counter_total(s: &str) -> Option<i64> {
|
||
let b = s.as_bytes();
|
||
for (i, &c) in b.iter().enumerate() {
|
||
if c != b'[' {
|
||
continue;
|
||
}
|
||
let a = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
let slash = i + 1 + a;
|
||
if a == 0 || b.get(slash) != Some(&b'/') {
|
||
continue;
|
||
}
|
||
let d = b[slash + 1..].iter().take_while(|c| c.is_ascii_digit()).count();
|
||
if d > 0 && b.get(slash + 1 + d) == Some(&b']') {
|
||
return js::parse_int(&s[slash + 1..slash + 1 + d]);
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
/// explored: a release in the sampled share of a rule's drops.
|
||
pub fn explored(key: &str, percent: f64) -> bool {
|
||
if percent <= 0.0 {
|
||
return false;
|
||
}
|
||
let d = Md5::digest(key.as_bytes());
|
||
let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]);
|
||
((n % 10_000) as f64) < percent * 100.0
|
||
}
|
||
|
||
static SIDECAR: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$"));
|
||
static EXECUTABLE: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.(exe|scr|com|pif|bat|cmd|vbs|lnk)SP*$"));
|
||
|
||
pub struct JudgedFile<'a> {
|
||
pub name: &'a str,
|
||
pub total_parts: u32,
|
||
pub groups: &'a [String],
|
||
pub first_message_id: &'a str,
|
||
}
|
||
|
||
pub enum Verdict {
|
||
Keep,
|
||
Drop { rule: &'static str, key: String, sample: bool },
|
||
}
|
||
|
||
/// RuleSet.judge with DEFAULT_RULES (no dropped signatures).
|
||
pub fn judge(release_key: &str, files: &[JudgedFile]) -> Verdict {
|
||
const DROPPED_TOOLS: [&str; 5] = ["ngPost", "hex32", "yuun", "JBinUp", "random"];
|
||
let sample = explored(release_key, 1.0);
|
||
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||
if !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||
let key = std::str::from_utf8(&c[1]).unwrap().to_lowercase();
|
||
return Verdict::Drop { rule: "executable", key, sample };
|
||
}
|
||
if files.iter().any(|f| !is_random_file(f.name)) {
|
||
return Verdict::Keep;
|
||
}
|
||
let first = &files[0];
|
||
let tool = upload_tool(first.first_message_id);
|
||
let group = first.groups.first().map(String::as_str).unwrap_or("");
|
||
if DROPPED_TOOLS.contains(&tool) {
|
||
return Verdict::Drop { rule: "garbage", key: format!("{tool} {group}"), sample };
|
||
}
|
||
if files.iter().all(|f| f.total_parts <= 3) {
|
||
return Verdict::Drop { rule: "per-article", key: tool.to_string(), sample };
|
||
}
|
||
Verdict::Keep
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn stems() {
|
||
assert_eq!(file_stem("x.mkv.vol03+04.par2"), "x");
|
||
assert_eq!(file_stem("x.part07.rar"), "x");
|
||
assert_eq!(file_stem("Movie.H.264.mkv.001"), "Movie.H.264");
|
||
assert_eq!(file_stem("a.7z.002"), "a");
|
||
}
|
||
|
||
#[test]
|
||
fn remainders() {
|
||
assert_eq!(subject_remainder(r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#), "Some Release");
|
||
assert!(names_release("Some Release"));
|
||
assert!(!names_release("elys de81bcabef5faf6a"));
|
||
}
|
||
|
||
#[test]
|
||
fn by_hand_as_the_regexes() {
|
||
let names = [
|
||
"x.mkv.vol03+04.par2", "x.part07.rar", "Movie.H.264.mkv.001", "a.7z.002", "a.7z", "b.zip.001", "B.ZIP", "c.r00", "c.r1234",
|
||
"d.vol1+2.par2", "d.vol+2.par2", ".par2", "e.tar.gz", "f.mkv", "g.m3u8", "h.bin.001", " spaced.rar ", "i.PART01.RAR", "j.part.rar",
|
||
"k.nfo.txt", "noext", "ünï.cödé.mkv.001", "l.ts.123", "m.r12.par2", "n.vol12-34.PAR2", "o.7z.1234", "p.123", "q.r1", ".rar.rar",
|
||
];
|
||
for n in names {
|
||
assert_eq!(file_stem(n), file_stem_re(n), "{n}");
|
||
}
|
||
let subjects = [
|
||
r#"Some.Release [01/45] - "x.rar" yEnc (1/50) 7168"#,
|
||
r#"[1/3] - "a.par2" yEnc (1/1) 123 "#,
|
||
r#"yEnc "x" yenc yEncx (1/2)"#,
|
||
r#"Show – S01E01 | 1080p : x "a" "b"#,
|
||
r#"name (12/3x) [4/] "q" YENC 12 34"#,
|
||
" lead\u{a0}nbsp yEnc\u{3000}99 ",
|
||
r#"elys-de81bcabef5faf6a - [Nyh5Q07Y] - "x.mkv" yEnc"#,
|
||
"a_yEnc b yEnc_c yEnc",
|
||
"yEnc yEnc 5",
|
||
"x yEnc 12 yEnc",
|
||
"",
|
||
];
|
||
for s in subjects {
|
||
assert_eq!(subject_remainder(s), subject_remainder_re(s), "{s}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn tools_by_hand_as_the_regexes() {
|
||
let ids = [
|
||
"FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@", "FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@news.example",
|
||
"b94268bc702040f7b811d5318f727e4f@ngPost", "b94268bc702040f7b811d5318f727e4f@b94268bc702040f7b811d5318f727e4f.yuun", "b94268bc702040f7b811d5318f727e4f@host-1.example",
|
||
"B94268bc702040f7b811d5318f727e4f@ngPost", "x@JBinUp.local", "y@JBinDown.local", "part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local", "PART12OF3.x",
|
||
"1790923856877$1729905757859$1@localhost", "179092385687$1$1@localhost", "t3Y4mV92YCp0McxDc5@zLZlsw.RYqes", "t3Y4mV92YC@zL.RYqes", "t3Y4mV92YCp0McxDc5@zLZlsw.RY.es",
|
||
"", "a@b", "b94268bc702040f7b811d5318f727e4f@", "b94268bc702040f7b811d5318f727e4f@host_x",
|
||
];
|
||
for id in ids {
|
||
assert_eq!(upload_tool(id), upload_tool_re(id), "{id}");
|
||
}
|
||
for t in ["Some Release", "a b", "ab-cd", "ab - c1", "ab.c", "x ab_12", "12 ab", "ab "] {
|
||
assert_eq!(name_like(t.as_bytes()), test(&NAME_LIKE, t), "{t}");
|
||
}
|
||
for t in ["[01/45] x", "x [1/] [2/30]", "[a/1] [3/4]", "no counter"] {
|
||
let re = FILE_COUNTER.captures(t.as_bytes()).and_then(|c| js::parse_int(std::str::from_utf8(&c[2]).unwrap()));
|
||
assert_eq!(file_counter_total(t), re, "{t}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn random_names() {
|
||
assert!(looks_random("prvjr7hapd0i1ny14pz4"));
|
||
assert!(looks_random("RmISakytFwXSCEzjVEpaKk"));
|
||
assert!(!looks_random("Some.Show.S01E01.1080p"));
|
||
assert!(looks_random("VIDEO_TS"));
|
||
}
|
||
|
||
#[test]
|
||
fn tools() {
|
||
assert_eq!(upload_tool("FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu"), "nyuu");
|
||
assert_eq!(upload_tool("b94268bc702040f7b811d5318f727e4f@ngPost"), "ngPost");
|
||
assert_eq!(upload_tool("x@y"), "other");
|
||
}
|
||
}
|