Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
- groups:add / ranges:plan / nodes:create / coordinator:run commands - the node API: lease, the range's chunks as one framed stream from the spool, batches, heartbeat, finish, release (node guard, token per node) - edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for the rest), posts a batch every 10 chunks, gives its range back on SIGTERM - releases are gathered per batch: 1.04 chunk rows per release at 10 chunks per batch (1.35 when built per XOVER chunk) - partitions are checked per batch instead of remembered Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
20910647fa
commit
d5f168d8d2
36 files changed
+1193
-14
No files matched your search
@@ -143,6 +143,7 @@ export class RangeIndexer {
|
||||
|
||||
/** What was found since the last take, and a fresh batch. */
|
||||
take(): EdgeBatch {
|
||||
this.judgeReleases()
|
||||
const out = this.batch
|
||||
out.counts = [...this.counts.values()]
|
||||
this.counts.clear()
|
||||
@@ -154,14 +155,22 @@ export class RangeIndexer {
|
||||
return this.aggregator.openFiles
|
||||
}
|
||||
|
||||
/** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
|
||||
private completed = new Map<string, AggregatedFile<IngestSegment>[]>()
|
||||
|
||||
private files(files: AggregatedFile<IngestSegment>[]) {
|
||||
const byRelease = new Map<string, AggregatedFile<IngestSegment>[]>()
|
||||
for (const file of files) {
|
||||
const key = releaseKeyOf(file.segments[0])
|
||||
const list = byRelease.get(key)
|
||||
const list = this.completed.get(key)
|
||||
if (list) list.push(file)
|
||||
else byRelease.set(key, [file])
|
||||
else this.completed.set(key, [file])
|
||||
}
|
||||
}
|
||||
|
||||
/** Judges and encodes the releases completed in this batch. */
|
||||
private judgeReleases() {
|
||||
const byRelease = this.completed
|
||||
this.completed = new Map()
|
||||
for (const [key, list] of byRelease) {
|
||||
const verdict = this.rules.judge(
|
||||
key,
|
||||
|
||||
Reference in new issue
Block a user