Rules drop posters at the header pass: the CelebMovie family (rule set version 3)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
093d0a71ad
commit
3d24db5565
9 files changed
+116
-9
No files matched your search
@@ -64,6 +64,8 @@ struct RulesJson {
|
||||
sample_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
drop_executables: Option<bool>,
|
||||
#[serde(default)]
|
||||
dropped_posters: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RulesJson {
|
||||
@@ -83,6 +85,9 @@ impl RulesJson {
|
||||
if let Some(e) = self.drop_executables {
|
||||
rules.drop_executables = e;
|
||||
}
|
||||
if let Some(posters) = self.dropped_posters {
|
||||
rules.dropped_posters = posters;
|
||||
}
|
||||
rules
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -752,7 +752,7 @@ impl RangeIndexer {
|
||||
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
||||
let judged: Vec<JudgedFile> = list
|
||||
.iter()
|
||||
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id })
|
||||
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster })
|
||||
.collect();
|
||||
match names::judge(&key, &judged, &self.rules) {
|
||||
Verdict::Keep => {
|
||||
|
||||
@@ -1241,6 +1241,7 @@ pub struct JudgedFile<'a> {
|
||||
pub total_parts: u32,
|
||||
pub groups: &'a [String],
|
||||
pub first_message_id: &'a str,
|
||||
pub poster: &'a str,
|
||||
}
|
||||
|
||||
pub enum Verdict {
|
||||
@@ -1260,6 +1261,8 @@ pub struct Rules {
|
||||
pub per_article_max_parts: u32,
|
||||
pub sample_percent: f64,
|
||||
pub drop_executables: bool,
|
||||
/// Fragments of the poster lowercased without punctuation (droppedPosters).
|
||||
pub dropped_posters: Vec<String>,
|
||||
}
|
||||
|
||||
impl Rules {
|
||||
@@ -1273,13 +1276,25 @@ impl Rules {
|
||||
per_article_max_parts: 3,
|
||||
sample_percent: 1.0,
|
||||
drop_executables: true,
|
||||
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// posterWords: lowercased, letters and digits only.
|
||||
pub fn poster_words(poster: &str) -> String {
|
||||
poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect()
|
||||
}
|
||||
|
||||
/// RuleSet.judge.
|
||||
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
||||
let sample = explored(release_key, rules.sample_percent);
|
||||
if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) {
|
||||
let words = poster_words(first.poster);
|
||||
if let Some(dropped) = rules.dropped_posters.iter().find(|p| words.contains(p.as_str())) {
|
||||
return Verdict::Drop { rule: "poster", key: dropped.clone(), sample };
|
||||
}
|
||||
}
|
||||
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||||
if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||||
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||||
@@ -1459,6 +1474,33 @@ mod numbered_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod poster_rule_tests {
|
||||
use super::{judge, JudgedFile, Rules, Verdict};
|
||||
|
||||
fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> {
|
||||
let groups = vec!["alt.binaries.erotica".to_string()];
|
||||
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster }];
|
||||
match judge("k", &files, rules) {
|
||||
Verdict::Keep => None,
|
||||
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dropped_posters_uploads_are_dropped() {
|
||||
let rules = Rules::defaults(Default::default());
|
||||
assert_eq!(verdict("@_CelebMovie_@", &rules), Some(("poster", "celebmovie".to_string())));
|
||||
assert_eq!(verdict("@__Celeb_Movie__@", &rules), Some(("poster", "celebmovie".to_string())));
|
||||
assert_eq!(verdict("NudeCelebs@CelebWow.com", &rules), Some(("poster", "celebwow".to_string())));
|
||||
assert_eq!(verdict("@nude-celebs", &rules), Some(("poster", "nudecelebs".to_string())));
|
||||
assert_eq!(verdict("celeborn@not.here (Celeborn)", &rules), None);
|
||||
assert_eq!(verdict(r#""CelebTA.com" <from@celebta.com>"#, &rules), None);
|
||||
let none = Rules { dropped_posters: vec![], ..Rules::defaults(Default::default()) };
|
||||
assert_eq!(verdict("@_CelebMovie_@", &none), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod image_set_tests {
|
||||
use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};
|
||||
|
||||
Reference in new issue
Block a user