Rules drop posters at the header pass: the CelebMovie family (rule set version 3)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
093d0a71ad
commit
3d24db5565
9 files changed
+116
-9
No files matched your search
@@ -83,7 +83,7 @@ class FragmentMerger {
|
|||||||
}
|
}
|
||||||
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
||||||
for (const [key, { files, rangeId }] of byRelease) {
|
for (const [key, { files, rangeId }] of byRelease) {
|
||||||
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId })))
|
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
|
||||||
if (verdict.rule !== null) continue
|
if (verdict.rule !== null) continue
|
||||||
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A rule set version with droppedPosters (lib/rules/rule_set.ts): the newest set plus the CelebMovie family, whose
|
||||||
|
* scraped celebrity pictures workers drop at the header pass from now on (LOG.md 2026-10-16 (65)). A new version, so the
|
||||||
|
* ranges record which ran with it. Nothing where there's no rule set yet (the default gets it) or the newest has it.
|
||||||
|
*/
|
||||||
|
const POSTERS = ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes']
|
||||||
|
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.defer(async (db) => {
|
||||||
|
const latest = await db.from('rule_sets').orderBy('version', 'desc').first()
|
||||||
|
if (!latest || 'droppedPosters' in latest.rules) return
|
||||||
|
await db.table('rule_sets').insert({
|
||||||
|
version: Number(latest.version) + 1,
|
||||||
|
rules: JSON.stringify({ ...latest.rules, droppedPosters: POSTERS }),
|
||||||
|
note: 'dropped posters: the CelebMovie family (2026-10-16)',
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {}
|
||||||
|
}
|
||||||
@@ -64,6 +64,8 @@ struct RulesJson {
|
|||||||
sample_percent: Option<f64>,
|
sample_percent: Option<f64>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
drop_executables: Option<bool>,
|
drop_executables: Option<bool>,
|
||||||
|
#[serde(default)]
|
||||||
|
dropped_posters: Option<Vec<String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RulesJson {
|
impl RulesJson {
|
||||||
@@ -83,6 +85,9 @@ impl RulesJson {
|
|||||||
if let Some(e) = self.drop_executables {
|
if let Some(e) = self.drop_executables {
|
||||||
rules.drop_executables = e;
|
rules.drop_executables = e;
|
||||||
}
|
}
|
||||||
|
if let Some(posters) = self.dropped_posters {
|
||||||
|
rules.dropped_posters = posters;
|
||||||
|
}
|
||||||
rules
|
rules
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -752,7 +752,7 @@ impl RangeIndexer {
|
|||||||
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
||||||
let judged: Vec<JudgedFile> = list
|
let judged: Vec<JudgedFile> = list
|
||||||
.iter()
|
.iter()
|
||||||
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id })
|
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster })
|
||||||
.collect();
|
.collect();
|
||||||
match names::judge(&key, &judged, &self.rules) {
|
match names::judge(&key, &judged, &self.rules) {
|
||||||
Verdict::Keep => {
|
Verdict::Keep => {
|
||||||
|
|||||||
@@ -1241,6 +1241,7 @@ pub struct JudgedFile<'a> {
|
|||||||
pub total_parts: u32,
|
pub total_parts: u32,
|
||||||
pub groups: &'a [String],
|
pub groups: &'a [String],
|
||||||
pub first_message_id: &'a str,
|
pub first_message_id: &'a str,
|
||||||
|
pub poster: &'a str,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub enum Verdict {
|
pub enum Verdict {
|
||||||
@@ -1260,6 +1261,8 @@ pub struct Rules {
|
|||||||
pub per_article_max_parts: u32,
|
pub per_article_max_parts: u32,
|
||||||
pub sample_percent: f64,
|
pub sample_percent: f64,
|
||||||
pub drop_executables: bool,
|
pub drop_executables: bool,
|
||||||
|
/// Fragments of the poster lowercased without punctuation (droppedPosters).
|
||||||
|
pub dropped_posters: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Rules {
|
impl Rules {
|
||||||
@@ -1273,13 +1276,25 @@ impl Rules {
|
|||||||
per_article_max_parts: 3,
|
per_article_max_parts: 3,
|
||||||
sample_percent: 1.0,
|
sample_percent: 1.0,
|
||||||
drop_executables: true,
|
drop_executables: true,
|
||||||
|
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// posterWords: lowercased, letters and digits only.
|
||||||
|
pub fn poster_words(poster: &str) -> String {
|
||||||
|
poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// RuleSet.judge.
|
/// RuleSet.judge.
|
||||||
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
||||||
let sample = explored(release_key, rules.sample_percent);
|
let sample = explored(release_key, rules.sample_percent);
|
||||||
|
if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) {
|
||||||
|
let words = poster_words(first.poster);
|
||||||
|
if let Some(dropped) = rules.dropped_posters.iter().find(|p| words.contains(p.as_str())) {
|
||||||
|
return Verdict::Drop { rule: "poster", key: dropped.clone(), sample };
|
||||||
|
}
|
||||||
|
}
|
||||||
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||||||
if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||||||
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||||||
@@ -1459,6 +1474,33 @@ mod numbered_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod poster_rule_tests {
|
||||||
|
use super::{judge, JudgedFile, Rules, Verdict};
|
||||||
|
|
||||||
|
fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> {
|
||||||
|
let groups = vec!["alt.binaries.erotica".to_string()];
|
||||||
|
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster }];
|
||||||
|
match judge("k", &files, rules) {
|
||||||
|
Verdict::Keep => None,
|
||||||
|
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_dropped_posters_uploads_are_dropped() {
|
||||||
|
let rules = Rules::defaults(Default::default());
|
||||||
|
assert_eq!(verdict("@_CelebMovie_@", &rules), Some(("poster", "celebmovie".to_string())));
|
||||||
|
assert_eq!(verdict("@__Celeb_Movie__@", &rules), Some(("poster", "celebmovie".to_string())));
|
||||||
|
assert_eq!(verdict("NudeCelebs@CelebWow.com", &rules), Some(("poster", "celebwow".to_string())));
|
||||||
|
assert_eq!(verdict("@nude-celebs", &rules), Some(("poster", "nudecelebs".to_string())));
|
||||||
|
assert_eq!(verdict("celeborn@not.here (Celeborn)", &rules), None);
|
||||||
|
assert_eq!(verdict(r#""CelebTA.com" <from@celebta.com>"#, &rules), None);
|
||||||
|
let none = Rules { dropped_posters: vec![], ..Rules::defaults(Default::default()) };
|
||||||
|
assert_eq!(verdict("@_CelebMovie_@", &none), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod image_set_tests {
|
mod image_set_tests {
|
||||||
use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};
|
use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ export interface ProbeCandidate {
|
|||||||
|
|
||||||
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
||||||
export interface RuleCount {
|
export interface RuleCount {
|
||||||
rule: 'crosspost' | 'obfuscated' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
||||||
key: string
|
key: string
|
||||||
articles: number
|
articles: number
|
||||||
files: number
|
files: number
|
||||||
@@ -215,7 +215,7 @@ export class RangeIndexer {
|
|||||||
uploadsOf(all).forEach((list, upload) => {
|
uploadsOf(all).forEach((list, upload) => {
|
||||||
const verdict = this.rules.judge(
|
const verdict = this.rules.judge(
|
||||||
key,
|
key,
|
||||||
list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId }))
|
list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster }))
|
||||||
)
|
)
|
||||||
if (verdict.rule === null) {
|
if (verdict.rule === null) {
|
||||||
this.batch.releases.push(chunkOf(key, list, upload))
|
this.batch.releases.push(chunkOf(key, list, upload))
|
||||||
|
|||||||
+1
-1
@@ -1,2 +1,2 @@
|
|||||||
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
|
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
|
||||||
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, type Rules, type Verdict } from './rule_set.js'
|
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'
|
||||||
+22
-3
@@ -24,6 +24,12 @@ export interface Rules {
|
|||||||
* 2026-10-09). Genuine software comes in archives.
|
* 2026-10-09). Genuine software comes in archives.
|
||||||
*/
|
*/
|
||||||
dropExecutables: boolean
|
dropExecutables: boolean
|
||||||
|
/**
|
||||||
|
* Posters whose uploads are dropped whatever they are: fragments of the poster lowercased without punctuation
|
||||||
|
* ("@__Celeb_Movie__@" → "celebmovie"). The CelebMovie family's scraped celebrity pictures, the same files again and
|
||||||
|
* again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)).
|
||||||
|
*/
|
||||||
|
droppedPosters: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||||
@@ -32,9 +38,15 @@ export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
|||||||
perArticleMaxParts: 3,
|
perArticleMaxParts: 3,
|
||||||
samplePercent: 1,
|
samplePercent: 1,
|
||||||
dropExecutables: true,
|
dropExecutables: true,
|
||||||
|
droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
|
||||||
}
|
}
|
||||||
|
|
||||||
export type Verdict = { rule: null } | { rule: 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
|
export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
|
||||||
|
|
||||||
|
/** A poster as droppedPosters matches it: lowercased, letters and digits only. */
|
||||||
|
export function posterWords(poster: string): string {
|
||||||
|
return poster.replace(/[^A-Za-z0-9]/g, '').toLowerCase()
|
||||||
|
}
|
||||||
|
|
||||||
/** Files that come along with a release's content: they don't make it one of executables or not. */
|
/** Files that come along with a release's content: they don't make it one of executables or not. */
|
||||||
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
|
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
|
||||||
@@ -57,18 +69,25 @@ export function ownerOf(groups: string[], priorities: Record<string, number>): s
|
|||||||
export class RuleSet {
|
export class RuleSet {
|
||||||
private signatures: Set<string>
|
private signatures: Set<string>
|
||||||
private tools: Set<string>
|
private tools: Set<string>
|
||||||
|
private posters: string[]
|
||||||
constructor(readonly rules: Rules) {
|
constructor(readonly rules: Rules) {
|
||||||
|
this.posters = rules.droppedPosters ?? []
|
||||||
this.signatures = new Set(rules.droppedSignatures)
|
this.signatures = new Set(rules.droppedSignatures)
|
||||||
this.tools = new Set(rules.droppedTools)
|
this.tools = new Set(rules.droppedTools)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A release's verdict from its files: dropped when its content is bare
|
* A release's verdict from its files: dropped when its poster is; when its content is bare
|
||||||
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
|
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
|
||||||
* dropped, as per-article random subjects when every file has few parts.
|
* dropped, as per-article random subjects when every file has few parts.
|
||||||
*/
|
*/
|
||||||
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict {
|
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict {
|
||||||
const sample = explored(releaseKey, this.rules.samplePercent)
|
const sample = explored(releaseKey, this.rules.samplePercent)
|
||||||
|
if (this.posters.length > 0 && files.length > 0) {
|
||||||
|
const words = posterWords(files[0].poster)
|
||||||
|
const dropped = this.posters.find((p) => words.includes(p))
|
||||||
|
if (dropped) return { rule: 'poster', key: dropped, sample }
|
||||||
|
}
|
||||||
if (this.rules.dropExecutables) {
|
if (this.rules.dropExecutables) {
|
||||||
const content = files.filter((f) => !SIDECAR.test(f.name))
|
const content = files.filter((f) => !SIDECAR.test(f.name))
|
||||||
if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
|
if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
|
||||||
|
|||||||
@@ -3,7 +3,13 @@ import { DEFAULT_RULES, RuleSet, ownerOf, uploadTool, explored } from '#lib/rule
|
|||||||
import { sevenZipVolume } from '#lib/files/index'
|
import { sevenZipVolume } from '#lib/files/index'
|
||||||
|
|
||||||
const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] })
|
const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] })
|
||||||
const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath') => ({ name, totalParts, groups: [group], firstMessageId })
|
const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath', poster = 'someone@example.com (Someone)') => ({
|
||||||
|
name,
|
||||||
|
totalParts,
|
||||||
|
groups: [group],
|
||||||
|
firstMessageId,
|
||||||
|
poster,
|
||||||
|
})
|
||||||
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
|
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
|
||||||
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'
|
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'
|
||||||
|
|
||||||
@@ -17,6 +23,17 @@ test.group('rules / owner', () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
test.group('rules / verdicts', () => {
|
test.group('rules / verdicts', () => {
|
||||||
|
test("a dropped poster's uploads are dropped whatever they are; others with the word in their name are not", ({ assert }) => {
|
||||||
|
const pics = (poster: string) => rules.judge('k', [file('!index0001.jpg', 1, NYUU, 'alt.binaries.erotica', poster)])
|
||||||
|
assert.include(pics('@_CelebMovie_@'), { rule: 'poster', key: 'celebmovie' })
|
||||||
|
assert.include(pics('@__Celeb_Movie__@'), { rule: 'poster', key: 'celebmovie' })
|
||||||
|
assert.include(pics('NudeCelebs@CelebWow.com'), { rule: 'poster', key: 'celebwow' })
|
||||||
|
assert.include(pics('@nude-celebs'), { rule: 'poster', key: 'nudecelebs' })
|
||||||
|
assert.deepEqual(pics('celeborn@not.here (Celeborn)'), { rule: null })
|
||||||
|
assert.deepEqual(pics('"CelebTA.com" <from@celebta.com>'), { rule: null })
|
||||||
|
assert.deepEqual(new RuleSet({ ...DEFAULT_RULES, droppedPosters: [], samplePercent: 0, priorities: {}, droppedSignatures: [] }).judge('k', [file('a.jpg', 1, NYUU, 'g', '@_CelebMovie_@')]), { rule: null })
|
||||||
|
})
|
||||||
|
|
||||||
test('a readable file keeps its whole release', ({ assert }) => {
|
test('a readable file keeps its whole release', ({ assert }) => {
|
||||||
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
|
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in new issue
Block a user