Rebuild: first end-to-end runs; prefetch per chunk, executable rule, wire fixes
Test and deploy / test (push) Successful in 20s
Test and deploy / deploy (push) Skipped

End to end on the workstation (coordinator:run, the API, edge:node, real
NNTP): a.b.e-book.french and a.b.cd.image.playstation2.dvdiso ranges.

- Prefetcher: every connection takes the next missing chunk (leased
  ranges first, in order); one connection per range (~26k lines/s) kept
  the worker waiting
- executable rule: releases of bare executables (malware floods named
  after films and software) are dropped; they were 92% of the catalog rows
  of the test ranges, 21% of the segments
- fragments per file (details once) and gzipped batch bodies; the JSON
  limit (1 MB by default) raised to 256 MB, empty strings kept (a rule
  count's key is '')
- release_chunks.batch_from is a bigint (boneless article numbers)
- tests get their own database; edge:bench --store times BatchStore:
  1.3 s for a boneless range (1.24 ms per release chunk)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 01:07:11 +02:00
1 parent d5f168d8d2
commit cd06f96f85
15 files changed
+201 -99

No files matched your search

+21 -4
View File
@@ -18,6 +18,12 @@ export interface Rules {
perArticleMaxParts: number
/** Share of each rule's drops kept in full as a sample (by a hash of the release key). */
samplePercent: number
/**
* Drop releases of bare executables (malware floods posted under film and
* software names; 92% of the catalog rows of the first test ranges, LOG.md
* 2026-10-09). Genuine software comes in archives.
*/
dropExecutables: boolean
}
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
@@ -25,9 +31,14 @@ export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'],
perArticleMaxParts: 3,
samplePercent: 1,
dropExecutables: true,
}
export type Verdict = { rule: null } | { rule: 'garbage' | 'per-article'; key: string; sample: boolean }
export type Verdict = { rule: null } | { rule: 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
/** Files that come along with a release's content: they don't make it one of executables or not. */
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
const EXECUTABLE = /\.(exe|scr|com|pif|bat|cmd|vbs|lnk)\s*$/i
export function isRandomFile(name: string) {
return looksRandom(fileStem(name))
@@ -52,15 +63,21 @@ export class RuleSet {
}
/**
* A release's verdict from its files: kept as a whole when any file has a
* readable name; else dropped as garbage when its signature or tool is
* A release's verdict from its files: dropped when its content is bare
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
* dropped, as per-article random subjects when every file has few parts.
*/
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict {
const sample = explored(releaseKey, this.rules.samplePercent)
if (this.rules.dropExecutables) {
const content = files.filter((f) => !SIDECAR.test(f.name))
if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
return { rule: 'executable', key: EXECUTABLE.exec(content[0].name)![1].toLowerCase(), sample }
}
}
if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
const first = files[0]
const tool = uploadTool(first.firstMessageId)
const sample = explored(releaseKey, this.rules.samplePercent)
// Garbage first: a dropped tool or signature isn't worth reading bodies for either.
const group = first.groups[0] ?? ''
if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample }