REBUILD.md: lone articles under scrambled names

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

REBUILD.md

@@ -774,7 +774,9 @@ is before frontier − `WINDOW_MS` are judged and written `finished`, as an
 it, else `finished: false, edge: true`. A file short of parts stays in
 its release and sets `partial` (the store: `complete = false`). A lone
 article of a multi-part random file is still counted `per-article`, not
-written. Gone: fragments and the cut-off, `uploads_of` (online now),
+written; before the block's end a scrambled name counts as random too
+(`looks_scrambled`: a random subject and poster per article, 160k of
+alt.binaries.storage's releases; LOG.md (122)). Gone: fragments and the cut-off, `uploads_of` (online now),
 `check.ts` and the fixture comparison (the parity with the TypeScript
 indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
 (incomplete releases are kept). The wire carries `finished`, `edge` and

app/services/prefetcher.ts

@@ -561,6 +561,8 @@ export class Prefetcher {
     const groups = new Map<string, number>()
     const candidates = await rows(db.query())
       .where('r.capture', true)
+      // A group taken off the list (groups.active) isn't worth the connections.
+      .where('g.active', true)
       .whereNull('r.stored_at')
       .where((q: any) =>
         q

edge/src/indexer.rs

@@ -5,7 +5,7 @@
 
 use crate::codec::{encode_chunk, ChunkFile};
 use crate::js::{self, sha1_hex};
-use crate::names::{self, is_random_file, JudgedFile, Release, Rules, Verdict};
+use crate::names::{self, is_random_file, looks_scrambled, JudgedFile, Release, Rules, Verdict};
 use rustc_hash::{FxHashMap, FxHashSet};
 use serde::{Deserialize, Serialize};
 use std::rc::Rc;
@@ -1176,11 +1176,11 @@ impl RangeIndexer {
         let t = std::time::Instant::now();
         let done = self.aggregator.add(segments);
         unsafe { PROF[3] += t.elapsed().as_nanos() as u64; }
-        self.files(done);
+        self.files(done, false);
         self.frontier.update();
         let t = std::time::Instant::now();
         let gone = self.aggregator.evict(self.frontier.closed_before(), self.idle, false);
-        self.files(gone);
+        self.files(gone, false);
         self.close(false);
         unsafe { PROF[4] += t.elapsed().as_nanos() as u64; }
         self.batch.next = next;
@@ -1191,7 +1191,7 @@ impl RangeIndexer {
     pub fn finish(&mut self) {
         self.frontier.settle();
         let gone = self.aggregator.evict(None, 0, true);
-        self.files(gone);
+        self.files(gone, true);
         self.close(true);
         self.batch.next = self.end + 1;
     }
@@ -1243,12 +1243,16 @@ impl RangeIndexer {
         out
     }
 
-    /// Finished files (complete, or the window closed on them) into the open release of their key and upload.
-    fn files(&mut self, files: Vec<Vec<Seg>>) {
+    /// Finished files (complete, or the window closed on them) into the open release of their key and upload; `at_end`
+    /// when the block ends on them.
+    fn files(&mut self, files: Vec<Vec<Seg>>, at_end: bool) {
         let batch = self.aggregator.batch;
         for file in files {
-            if file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1) && is_random_file(&file[0].file.name) {
-                // One article of a multi-part random file (its upload per article: ngPost's, nyuu's): not a release.
+            let lone = file.len() == 1 && (file[0].number != 1 || file[0].file.total_parts > 1);
+            // A scrambled name only before the block's end, where the rest of a file can still be in the next block.
+            if lone && (is_random_file(&file[0].file.name) || (!at_end && looks_scrambled(&file[0].file.name))) {
+                // One article of a multi-part random file (its upload per article: ngPost's, nyuu's, and a random poster's
+                // per article, 160k in alt.binaries.storage; LOG.md (122)): not a release.
                 let tool = names::upload_tool(&file[0].message_id);
                 self.count("per-article", tool.to_string(), 1, 0);
                 continue;

edge/src/names.rs

@@ -1065,6 +1065,22 @@ pub fn is_random_file(name: &str) -> bool {
     looks_random(&file_stem(name))
 }
 
+/// looksScrambled: at least half the letters and digits of a file's stem in tokens of 5 or more that switch case
+/// (lower to upper) or between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa").
+/// The names a random-subject-per-article upload gives each article that looksRandom misses for their separators.
+pub fn looks_scrambled(name: &str) -> bool {
+    let stem = file_stem(name);
+    let (mut total, mut scrambled) = (0, 0);
+    for token in stem.as_bytes().split(|c| !c.is_ascii_alphanumeric()).filter(|t| !t.is_empty()) {
+        total += token.len();
+        let switches = token.windows(2).filter(|w| (w[0].is_ascii_lowercase() && w[1].is_ascii_uppercase()) || w[0].is_ascii_digit() != w[1].is_ascii_digit()).count();
+        if token.len() >= 5 && switches >= 3 {
+            scrambled += token.len();
+        }
+    }
+    total >= 8 && scrambled * 2 >= total
+}
+
 static SEVEN_SPLIT: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z\.([0-9]{1,4})$"));
 static SEVEN: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.7z$"));
 static PART1: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\.part0*1\.rar$"));
@@ -1376,6 +1392,17 @@ mod tests {
         assert_eq!(file_stem_re("Morning Lovers.wmv.058"), "Morning Lovers");
     }
 
+    #[test]
+    fn scrambled_names() {
+        for n in ["nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa", "PNSVWSV-xLaNv6o5qS840e3lwSZyH", "Biq-YsXu8G33ktt"] {
+            assert!(looks_scrambled(n), "{n}");
+        }
+        // Short ones of the spam too (8% of alt.binaries.storage's): too few switches to tell.
+        for n in ["A1Monitor v6.2.2", "PDF2Word v1.3", "Some.Show.S01E01.1080p.WEB.part1.rar", "MacGyver.S01E01.mkv", "Jimmy and the Crawler (mobi)", "Q25m.PDUYtbA", "82WH.2iuTK"] {
+            assert!(!looks_scrambled(n), "{n}");
+        }
+    }
+
     #[test]
     fn display_names() {
         let porn4you = "webmaster@hdbangvidz.com (Porn4You)";

lib/edge/range_indexer.ts

@@ -1,7 +1,7 @@
 import { hash } from 'node:crypto'
 import type { Overview } from '../nntp/index.js'
 import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
-import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js'
+import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
 import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
 import { encodeChunk, type ChunkFile } from '../chunks/index.js'
 
@@ -280,8 +280,10 @@ export class RangeIndexer {
     }
     for (const list of byFile.values()) {
       if (this.options.partialFiles) {
-        // As the Rust window's files(): one article of a multi-part random file is no release.
-        if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && isRandomFile(list[0].file.name)) {
+        // As the Rust window's files(): one article of a multi-part random file is no release; a scrambled name only
+        // before the end, where the rest of the file can still come.
+        const name = list[0].file.name
+        if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && (isRandomFile(name) || (!atEnd && isScrambledFile(name)))) {
           this.count('per-article', uploadTool(list[0].messageId), 1, 0)
           continue
         }

lib/rules/index.ts

@@ -1,2 +1,2 @@
 export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
-export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'
+export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'

lib/rules/rule_set.ts

@@ -56,6 +56,26 @@ export function isRandomFile(name: string) {
   return looksRandom(fileStem(name))
 }
 
+/**
+ * At least half the letters and digits of a file's stem in tokens of 5 or more that switch case (lower to upper) or
+ * between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"): the names a
+ * random-subject-per-article upload gives each article that looksRandom misses for their separators.
+ */
+export function isScrambledFile(name: string) {
+  let total = 0
+  let scrambled = 0
+  for (const token of fileStem(name).split(/[^A-Za-z0-9]+/)) {
+    total += token.length
+    let switches = 0
+    for (let i = 1; i < token.length; i++) {
+      const [a, b] = [token[i - 1], token[i]]
+      if ((/[a-z]/.test(a) && /[A-Z]/.test(b)) || /\d/.test(a) !== /\d/.test(b)) switches++
+    }
+    if (token.length >= 5 && switches >= 3) scrambled += token.length
+  }
+  return total >= 8 && scrambled * 2 >= total
+}
+
 /** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
 export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
   let owner: string | null = null

tests/unit/edge/range_indexer.spec.ts

@@ -105,6 +105,20 @@ test.group('edge / RangeIndexer', () => {
     assert.deepEqual(batch.fragments, [])
     assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
   })
+
+  test('partialFiles: one article of a multi-part file under a scrambled name is per-article before the end, a partial release at it', ({ assert }) => {
+    const indexer = new RangeIndexer({ group: GROUP, start: 100, end: 1_000_000, rules, idleChunks: 1, edgeArticles: 0, partialFiles: true })
+    // Part 8 of 67, a random poster's: given up mid-range.
+    indexer.add(post(100, 'nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr', 67, { poster: 'guAym5TqSlp SCTLKQ0R <8nGc3Rk@OrjrYqFkp.z>' }).slice(7, 8), 200)
+    indexer.add([], 300)
+    indexer.add([], 400)
+    // The same shape still open when the block ends: the rest can be in the next one.
+    indexer.add(post(999_990, 'bslXlwgPiqZadhRa', 5).slice(1, 2), 1_000_001)
+    indexer.finish()
+    const batch = indexer.take()
+    assert.deepEqual(batch.releases.map((r) => [r.name, r.partial]), [['bslXlwgPiqZadhRa', true]])
+    assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['per-article', 1, 0]])
+  })
 })
 
 test.group('edge / taking over a range', () => {