Rebuild: the batch store (catalog upserts, write-once chunks, candidates, rule counts, fragments, resume point)
A batch is stored in one transaction and only at its range's resume point, so a retried batch can't count twice. Garbage is judged before per-article: a dropped tool isn't worth reading bodies for either. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0714b2c81a
commit
20910647fa
6 files changed
+242
-7
No files matched your search
@@ -20,6 +20,9 @@ export interface ReleaseChunk {
|
||||
bytes: number
|
||||
firstArticle: number
|
||||
lastArticle: number
|
||||
/** A file with a readable name; a par2 file. */
|
||||
readable: boolean
|
||||
hasPar2: boolean
|
||||
payload: Buffer
|
||||
}
|
||||
|
||||
@@ -63,6 +66,8 @@ export interface Fragment {
|
||||
}
|
||||
|
||||
export interface EdgeBatch {
|
||||
/** The range's resume point this batch starts at. */
|
||||
from: number
|
||||
/** First article number not covered yet. */
|
||||
next: number
|
||||
headers: number
|
||||
@@ -193,8 +198,8 @@ export class RangeIndexer {
|
||||
}
|
||||
}
|
||||
|
||||
private empty(next: number): EdgeBatch {
|
||||
return { next, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
|
||||
private empty(from: number): EdgeBatch {
|
||||
return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,6 +252,8 @@ function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseCh
|
||||
bytes,
|
||||
firstArticle,
|
||||
lastArticle,
|
||||
readable: chunkFiles.some((f) => !isRandomFile(f.name)),
|
||||
hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
|
||||
payload: encodeChunk(chunkFiles),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user