diff --git a/app/services/prefetcher.ts b/app/services/prefetcher.ts index dbdb0d9..0300d0f 100644 --- a/app/services/prefetcher.ts +++ b/app/services/prefetcher.ts @@ -561,6 +561,8 @@ export class Prefetcher { const groups = new Map() const candidates = await rows(db.query()) .where('r.capture', true) + // A group taken off the list (groups.active) isn't worth the connections. + .where('g.active', true) .whereNull('r.stored_at') .where((q: any) => q diff --git a/edge/src/indexer.rs b/edge/src/indexer.rs index a51c700..f50a783 100644 --- a/edge/src/indexer.rs +++ b/edge/src/indexer.rs @@ -5,7 +5,7 @@ use crate::codec::{encode_chunk, ChunkFile}; use crate::js::{self, sha1_hex}; -use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict}; +use crate::names::{self, is_random_file, looks_scrambled, JudgedFile, Release, Rules, Verdict}; use rustc_hash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; use std::rc::Rc; @@ -1176,11 +1176,11 @@ impl RangeIndexer { let t = std::time::Instant::now(); let done = self.aggregator.add(segments); unsafe { PROF[3] += t.elapsed().as_nanos() as u64; } - self.files(done); + self.files(done, false); self.frontier.update(); let t = std::time::Instant::now(); let gone = self.aggregator.evict(self.frontier.closed_before(), self.idle, false); - self.files(gone); + self.files(gone, false); self.close(false); unsafe { PROF[4] += t.elapsed().as_nanos() as u64; } self.batch.next = next; @@ -1191,7 +1191,7 @@ impl RangeIndexer { pub fn finish(&mut self) { self.frontier.settle(); let gone = self.aggregator.evict(None, 0, true); - self.files(gone); + self.files(gone, true); self.close(true); self.batch.next = self.end + 1; } @@ -1243,12 +1243,16 @@ impl RangeIndexer { out } - /// Finished files (complete, or the window closed on them) into the open release of their key and upload. - fn files(&mut self, files: Vec>) { + /// Finished files (complete, or the window closed on them) into the open release of their key and upload; `at_end` + /// when the block ends on them. + fn files(&mut self, files: Vec>, at_end: bool) { let batch = self.aggregator.batch; for file in files { - if file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1) && is_random_file(&file[0].file.name) { - // One article of a multi-part random file (its upload per article: ngPost's, nyuu's): not a release. + let lone = file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1); + // A scrambled name only before the block's end, where the rest of a file can still be in the next block. + if lone && (is_random_file(&file[0].file.name) || (!at_end && looks_scrambled(&file[0].file.name))) { + // One article of a multi-part random file (its upload per article: ngPost's, nyuu's, and a random poster's + // per article, 160k in alt.binaries.storage; LOG.md (122)): not a release. let tool = names::upload_tool(&file[0].message_id); self.count("per-article", tool.to_string(), 1, 0); continue; diff --git a/edge/src/names.rs b/edge/src/names.rs index 8879f6a..bd95e48 100644 --- a/edge/src/names.rs +++ b/edge/src/names.rs @@ -1065,6 +1065,22 @@ pub fn is_random_file(name: &str) -> bool { looks_random(&file_stem(name)) } +/// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case +/// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"). +/// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators. +pub fn looks_scrambled(name: &str) -> bool { + let stem = file_stem(name); + let (mut total, mut scrambled) = (0, 0); + for token in stem.as_bytes().split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) { + total += token.len(); + let switches = token.windows(2).filter(|w| (w[0].is_ascii_lowercase() && w[1].is_ascii_uppercase()) || w[0].is_ascii_digit() != w[1].is_ascii_digit()).count(); + if token.len() >= 5 && switches >= 3 { + scrambled += token.len(); + } + } + total >= 8 && scrambled * 2 >= total +} + static SEVEN_SPLIT: LazyLock = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$")); static SEVEN: LazyLock = LazyLock::new(|| re(r"(?i)\.7z$")); static PART1: LazyLock = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$")); @@ -1376,6 +1392,17 @@ mod tests { assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers"); } + #[test] + fn scrambled_names() { + for n in ["nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa", "PNSVWSV-xLaNv6o5qS840e3lwSZyH", "Biq-YsXu8G33ktt"] { + assert!(looks_scrambled(n), "{n}"); + } + // Short ones of the spam too (8% of alt.binaries.storage's): too few switches to tell. + for n in ["A1Monitor v6.2.2", "PDF2Word v1.3", "Some.Show.S01E01.1080p.WEB.part1.rar", "MacGyver.S01E01.mkv", "Jimmy and the Crawler (mobi)", "Q25m.PDUYtbA", "82WH.2iuTK"] { + assert!(!looks_scrambled(n), "{n}"); + } + } + #[test] fn display_names() { let porn4you = "webmaster@hdbangvidz.com (Porn4You)"; diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 6dca98b..9f5add3 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -1,7 +1,7 @@ import { hash } from 'node:crypto' import type { Overview } from '../nntp/index.js' import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js' -import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js' +import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js' import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js' import { encodeChunk, type ChunkFile } from '../chunks/index.js' @@ -280,8 +280,10 @@ export class RangeIndexer { } for (const list of byFile.values()) { if (this.options.partialFiles) { - // As the Rust window's files(): one article of a multi-part random file is no release. - if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && isRandomFile(list[0].file.name)) { + // As the Rust window's files(): one article of a multi-part random file is no release; a scrambled name only + // before the end, where the rest of the file can still come. + const name = list[0].file.name + if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && (isRandomFile(name) || (!atEnd && isScrambledFile(name)))) { this.count('per-article', uploadTool(list[0].messageId), 1, 0) continue } diff --git a/lib/rules/index.ts b/lib/rules/index.ts index f917b64..0906a78 100644 --- a/lib/rules/index.ts +++ b/lib/rules/index.ts @@ -1,2 +1,2 @@ export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js' -export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js' +export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js' diff --git a/lib/rules/rule_set.ts b/lib/rules/rule_set.ts index 09b5f2f..65cf6cb 100644 --- a/lib/rules/rule_set.ts +++ b/lib/rules/rule_set.ts @@ -56,6 +56,26 @@ export function isRandomFile(name: string) { return looksRandom(fileStem(name)) } +/** + * At least half the letters and digits of a file's stem in tokens of 5 or more that switch case (lower to upper) or + * between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"): the names a + * random-subject-per-article upload gives each article that looksRandom misses for their separators. + */ +export function isScrambledFile(name: string) { + let total = 0 + let scrambled = 0 + for (const token of fileStem(name).split(/[^A-Za-z0-9]+/)) { + total += token.length + let switches = 0 + for (let i = 1; i < token.length; i++) { + const [a, b] = [token[i - 1], token[i]] + if ((/[a-z]/.test(a) && /[A-Z]/.test(b)) || /\d/.test(a) !== /\d/.test(b)) switches++ + } + if (token.length >= 5 && switches >= 3) scrambled += token.length + } + return total >= 8 && scrambled * 2 >= total +} + /** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */ export function ownerOf(groups: string[], priorities: Record): string | null { let owner: string | null = null diff --git a/tests/unit/edge/range_indexer.spec.ts b/tests/unit/edge/range_indexer.spec.ts index 0127bb0..0c6d630 100644 --- a/tests/unit/edge/range_indexer.spec.ts +++ b/tests/unit/edge/range_indexer.spec.ts @@ -105,6 +105,20 @@ test.group('edge / RangeIndexer', () => { assert.deepEqual(batch.fragments, []) assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]]) }) + + test('partialFiles: one article of a multi-part file under a scrambled name is per-article before the end, a partial release at it', ({ assert }) => { + const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 1_000_000, rules, idleChunks: 1, edgeArticles: 0, partialFiles: true }) + // Part 8 of 67, a random poster's: given up mid-range. + indexer.add(post(100, 'nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr', 67, { poster: 'guAym5TqSlp SCTLKQ0R <8nGc3Rk@OrjrYqFkp.z>' }).slice(7, 8), 200) + indexer.add([], 300) + indexer.add([], 400) + // The same shape still open when the block ends: the rest can be in the next one. + indexer.add(post(999_990, 'bslXlwgPiqZadhRa', 5).slice(1, 2), 1_000_001) + indexer.finish() + const batch = indexer.take() + assert.deepEqual(batch.releases.map((r) => [r.name, r.partial]), [['bslXlwgPiqZadhRa', true]]) + assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]]) + }) }) test.group('edge / taking over a range', () => {