Rebuild: the batch store (catalog upserts, write-once chunks, candidates, rule counts, fragments, resume point)
Test and deploy / test (push) Successful in 20s
Test and deploy / deploy (push) Skipped

A batch is stored in one transaction and only at its range's resume
point, so a retried batch can't count twice. Garbage is judged before
per-article: a dropped tool isn't worth reading bodies for either.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:21:17 +02:00
1 parent 0714b2c81a
commit 20910647fa
6 files changed
+242 -7

No files matched your search

+9 -2
View File
@@ -20,6 +20,9 @@ export interface ReleaseChunk {
bytes: number
firstArticle: number
lastArticle: number
/** A file with a readable name; a par2 file. */
readable: boolean
hasPar2: boolean
payload: Buffer
}
@@ -63,6 +66,8 @@ export interface Fragment {
}
export interface EdgeBatch {
/** The range's resume point this batch starts at. */
from: number
/** First article number not covered yet. */
next: number
headers: number
@@ -193,8 +198,8 @@ export class RangeIndexer {
}
}
private empty(next: number): EdgeBatch {
return { next, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
private empty(from: number): EdgeBatch {
return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
}
}
@@ -247,6 +252,8 @@ function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseCh
bytes,
firstArticle,
lastArticle,
readable: chunkFiles.some((f) => !isRandomFile(f.name)),
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
payload: encodeChunk(chunkFiles),
}
}