Rebuild: first end-to-end runs; prefetch per chunk, executable rule, wire fixes
Test and deploy / test (push) Successful in 20s
Test and deploy / deploy (push) Skipped

End to end on the workstation (coordinator:run, the API, edge:node, real
NNTP): a.b.e-book.french and a.b.cd.image.playstation2.dvdiso ranges.

- Prefetcher: every connection takes the next missing chunk (leased
  ranges first, in order); one connection per range (~26k lines/s) kept
  the worker waiting
- executable rule: releases of bare executables (malware floods named
  after films and software) are dropped; they were 92% of the catalog rows
  of the test ranges, 21% of the segments
- fragments per file (details once) and gzipped batch bodies; the JSON
  limit (1 MB by default) raised to 256 MB, empty strings kept (a rule
  count's key is '')
- release_chunks.batch_from is a bigint (boneless article numbers)
- tests get their own database; edge:bench --store times BatchStore:
  1.3 s for a boneless range (1.24 ms per release chunk)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 01:07:11 +02:00
1 parent d5f168d8d2
commit cd06f96f85
15 files changed
+201 -99

No files matched your search

+24 -14
View File
@@ -43,13 +43,17 @@ export interface ProbeCandidate {
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
export interface RuleCount {
rule: 'crosspost' | 'obfuscated' | 'garbage' | 'per-article' | 'incomplete'
rule: 'crosspost' | 'obfuscated' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
key: string
articles: number
files: number
}
/** A segment of a file cut off by the range's edges, for the neighbouring range. */
/**
* A file cut off by the range's edges, for the neighbouring range: its
* details once, and the segments this range had, as
* [part, message-id, bytes, article number].
*/
export interface Fragment {
fileKey: string
name: string
@@ -58,11 +62,8 @@ export interface Fragment {
totalParts: number
groups: string[]
release: { key: string; name: string; source: string; fileTotal?: number } | null
number: number
messageId: string
bytes: number
articleNumber: number
postedAt: number | null
segments: [number, string, number, number][]
}
export interface EdgeBatch {
@@ -186,13 +187,25 @@ export class RangeIndexer {
}
}
/** Given-up segments: kept near the range's edges (the neighbour may complete them), counted otherwise. */
/** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */
private cutOff(segments: IngestSegment[]) {
const { start, end } = this.options
const byFile = new Map<string, Fragment>()
for (const s of segments) {
const n = Number(s.articleNumber)
if (Number.isFinite(n) && (n < start + this.edge || n > end - this.edge)) this.batch.fragments.push(fragmentOf(s))
else this.count('incomplete', '', 1, 0)
if (!Number.isFinite(n) || (n >= start + this.edge && n <= end - this.edge)) {
this.count('incomplete', '', 1, 0)
continue
}
let fragment = byFile.get(s.fileKey)
if (!fragment) {
fragment = fragmentOf(s)
byFile.set(s.fileKey, fragment)
this.batch.fragments.push(fragment)
}
fragment.segments.push([s.number, s.messageId, s.bytes, n])
const t = s.postedAt?.getTime()
if (t !== undefined && Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
}
}
@@ -301,10 +314,7 @@ function fragmentOf(s: IngestSegment): Fragment {
totalParts: s.totalParts,
groups: s.groups,
release: s.release ? { key: s.release.key, name: s.release.name, source: s.release.source, ...(s.release.fileTotal ? { fileTotal: s.release.fileTotal } : {}) } : null,
number: s.number,
messageId: s.messageId,
bytes: s.bytes,
articleNumber: Number(s.articleNumber),
postedAt: s.postedAt && Number.isFinite(s.postedAt.getTime()) ? s.postedAt.getTime() : null,
postedAt: null,
segments: [],
}
}