Lone articles under scrambled names are per-article; inactive groups get no corpus fill
Test and deploy / test (push) Successful in 3m6s
Test and deploy / deploy (push) Skipped

One article of a multi-part file under a name like "nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr" or
"bslXlwgPiqZadhRa" (a random subject and poster per article, with fake counters) was a release:
looksRandom misses these names for their separators. 160k of alt.binaries.storage's 162k
non-Bob releases, and about 60% of all single-article releases in a 2% sample.

looks_scrambled / isScrambledFile: at least half the stem's letters and digits in tokens of 5+
that switch case or letter/digit 3 times or more (93% of the storage spam). Used for lone
articles given up before the block's end only; at the end the strict looksRandom stays, as the
rest of such a file can be in the next block. The namer's re-index (partialFiles) does the same.

The prefetcher's corpus fill skips groups that aren't active.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-29 22:19:11 +01:00
1 parent 5132ab9083
commit cfce8884d4
7 files changed
+81 -12

No files matched your search

+5 -3
View File
@@ -1,7 +1,7 @@
import { hash } from 'node:crypto'
import type { Overview } from '../nntp/index.js'
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js'
import { RuleSet, isRandomFile, isScrambledFile, uploadTool, type Rules } from '../rules/index.js'
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
import { encodeChunk, type ChunkFile } from '../chunks/index.js'
@@ -280,8 +280,10 @@ export class RangeIndexer {
}
for (const list of byFile.values()) {
if (this.options.partialFiles) {
// As the Rust window's files(): one article of a multi-part random file is no release.
if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && isRandomFile(list[0].file.name)) {
// As the Rust window's files(): one article of a multi-part random file is no release; a scrambled name only
// before the end, where the rest of the file can still come.
const name = list[0].file.name
if (list.length === 1 && (list[0].number !== 1 || list[0].totalParts > 1) && (isRandomFile(name) || (!atEnd && isScrambledFile(name)))) {
this.count('per-article', uploadTool(list[0].messageId), 1, 0)
continue
}
+1 -1
View File
@@ -1,2 +1,2 @@
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'
export { RuleSet, DEFAULT_RULES, isRandomFile, isScrambledFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'
+20
View File
@@ -56,6 +56,26 @@ export function isRandomFile(name: string) {
return looksRandom(fileStem(name))
}
/**
* At least half the letters and digits of a file's stem in tokens of 5 or more that switch case (lower to upper) or
* between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"): the names a
* random-subject-per-article upload gives each article that looksRandom misses for their separators.
*/
export function isScrambledFile(name: string) {
let total = 0
let scrambled = 0
for (const token of fileStem(name).split(/[^A-Za-z0-9]+/)) {
total += token.length
let switches = 0
for (let i = 1; i < token.length; i++) {
const [a, b] = [token[i - 1], token[i]]
if ((/[a-z]/.test(a) && /[A-Z]/.test(b)) || /\d/.test(a) !== /\d/.test(b)) switches++
}
if (token.length >= 5 && switches >= 3) scrambled += token.length
}
return total >= 8 && scrambled * 2 >= total
}
/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
let owner: string | null = null