Rebuild: first end-to-end runs; prefetch per chunk, executable rule, wire fixes
End to end on the workstation (coordinator:run, the API, edge:node, real NNTP): a.b.e-book.french and a.b.cd.image.playstation2.dvdiso ranges. - Prefetcher: every connection takes the next missing chunk (leased ranges first, in order); one connection per range (~26k lines/s) kept the worker waiting - executable rule: releases of bare executables (malware floods named after films and software) are dropped; they were 92% of the catalog rows of the test ranges, 21% of the segments - fragments per file (details once) and gzipped batch bodies; the JSON limit (1 MB by default) raised to 256 MB, empty strings kept (a rule count's key is '') - release_chunks.batch_from is a bigint (boneless article numbers) - tests get their own database; edge:bench --store times BatchStore: 1.3 s for a boneless range (1.24 ms per release chunk) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d5f168d8d2
commit
cd06f96f85
15 files changed
+201
-99
No files matched your search
+24
-14
@@ -43,13 +43,17 @@ export interface ProbeCandidate {
|
||||
|
||||
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
||||
export interface RuleCount {
|
||||
rule: 'crosspost' | 'obfuscated' | 'garbage' | 'per-article' | 'incomplete'
|
||||
rule: 'crosspost' | 'obfuscated' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
||||
key: string
|
||||
articles: number
|
||||
files: number
|
||||
}
|
||||
|
||||
/** A segment of a file cut off by the range's edges, for the neighbouring range. */
|
||||
/**
|
||||
* A file cut off by the range's edges, for the neighbouring range: its
|
||||
* details once, and the segments this range had, as
|
||||
* [part, message-id, bytes, article number].
|
||||
*/
|
||||
export interface Fragment {
|
||||
fileKey: string
|
||||
name: string
|
||||
@@ -58,11 +62,8 @@ export interface Fragment {
|
||||
totalParts: number
|
||||
groups: string[]
|
||||
release: { key: string; name: string; source: string; fileTotal?: number } | null
|
||||
number: number
|
||||
messageId: string
|
||||
bytes: number
|
||||
articleNumber: number
|
||||
postedAt: number | null
|
||||
segments: [number, string, number, number][]
|
||||
}
|
||||
|
||||
export interface EdgeBatch {
|
||||
@@ -186,13 +187,25 @@ export class RangeIndexer {
|
||||
}
|
||||
}
|
||||
|
||||
/** Given-up segments: kept near the range's edges (the neighbour may complete them), counted otherwise. */
|
||||
/** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */
|
||||
private cutOff(segments: IngestSegment[]) {
|
||||
const { start, end } = this.options
|
||||
const byFile = new Map<string, Fragment>()
|
||||
for (const s of segments) {
|
||||
const n = Number(s.articleNumber)
|
||||
if (Number.isFinite(n) && (n < start + this.edge || n > end - this.edge)) this.batch.fragments.push(fragmentOf(s))
|
||||
else this.count('incomplete', '', 1, 0)
|
||||
if (!Number.isFinite(n) || (n >= start + this.edge && n <= end - this.edge)) {
|
||||
this.count('incomplete', '', 1, 0)
|
||||
continue
|
||||
}
|
||||
let fragment = byFile.get(s.fileKey)
|
||||
if (!fragment) {
|
||||
fragment = fragmentOf(s)
|
||||
byFile.set(s.fileKey, fragment)
|
||||
this.batch.fragments.push(fragment)
|
||||
}
|
||||
fragment.segments.push([s.number, s.messageId, s.bytes, n])
|
||||
const t = s.postedAt?.getTime()
|
||||
if (t !== undefined && Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
|
||||
}
|
||||
}
|
||||
|
||||
@@ -301,10 +314,7 @@ function fragmentOf(s: IngestSegment): Fragment {
|
||||
totalParts: s.totalParts,
|
||||
groups: s.groups,
|
||||
release: s.release ? { key: s.release.key, name: s.release.name, source: s.release.source, ...(s.release.fileTotal ? { fileTotal: s.release.fileTotal } : {}) } : null,
|
||||
number: s.number,
|
||||
messageId: s.messageId,
|
||||
bytes: s.bytes,
|
||||
articleNumber: Number(s.articleNumber),
|
||||
postedAt: s.postedAt && Number.isFinite(s.postedAt.getTime()) ? s.postedAt.getTime() : null,
|
||||
postedAt: null,
|
||||
segments: [],
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user