Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
Test and deploy / test (push) Successful in 21s
Test and deploy / deploy (push) Skipped

- groups:add / ranges:plan / nodes:create / coordinator:run commands
- the node API: lease, the range's chunks as one framed stream from the
  spool, batches, heartbeat, finish, release (node guard, token per node)
- edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for
  the rest), posts a batch every 10 chunks, gives its range back on SIGTERM
- releases are gathered per batch: 1.04 chunk rows per release at 10
  chunks per batch (1.35 when built per XOVER chunk)
- partitions are checked per batch instead of remembered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:31:06 +02:00
1 parent 20910647fa
commit d5f168d8d2
36 files changed
+1193 -14

No files matched your search

+12 -3
View File
@@ -143,6 +143,7 @@ export class RangeIndexer {
/** What was found since the last take, and a fresh batch. */
take(): EdgeBatch {
this.judgeReleases()
const out = this.batch
out.counts = [...this.counts.values()]
this.counts.clear()
@@ -154,14 +155,22 @@ export class RangeIndexer {
return this.aggregator.openFiles
}
/** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
private completed = new Map<string, AggregatedFile<IngestSegment>[]>()
private files(files: AggregatedFile<IngestSegment>[]) {
const byRelease = new Map<string, AggregatedFile<IngestSegment>[]>()
for (const file of files) {
const key = releaseKeyOf(file.segments[0])
const list = byRelease.get(key)
const list = this.completed.get(key)
if (list) list.push(file)
else byRelease.set(key, [file])
else this.completed.set(key, [file])
}
}
/** Judges and encodes the releases completed in this batch. */
private judgeReleases() {
const byRelease = this.completed
this.completed = new Map()
for (const [key, list] of byRelease) {
const verdict = this.rules.judge(
key,