Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
Test and deploy / test (push) Successful in 21s
Test and deploy / deploy (push) Skipped

- groups:add / ranges:plan / nodes:create / coordinator:run commands
- the node API: lease, the range's chunks as one framed stream from the
  spool, batches, heartbeat, finish, release (node guard, token per node)
- edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for
  the rest), posts a batch every 10 chunks, gives its range back on SIGTERM
- releases are gathered per batch: 1.04 chunk rows per release at 10
  chunks per batch (1.35 when built per XOVER chunk)
- partitions are checked per batch instead of remembered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:31:06 +02:00
1 parent 20910647fa
commit d5f168d8d2
36 files changed
+1193 -14

No files matched your search

+10
View File
@@ -20,6 +20,16 @@ const authConfig = defineConfig({
}),
}),
/**
* Edge workers (REBUILD.md, part 2): a token per node.
*/
node: tokensGuard({
provider: tokensUserProvider({
tokens: 'accessTokens',
model: () => import('#models/node'),
}),
}),
/**
* Session-based guard for browser authentication.
*/
+2 -1
View File
@@ -66,7 +66,8 @@ const bodyParserConfig = defineConfig({
/**
* Maximum accepted payload size for multipart requests.
*/
limit: '20mb',
// Edge worker batches carry their releases' chunks (a few MB, base64).
limit: '64mb',
/**
* Content types handled by the multipart parser.
+24
View File
@@ -0,0 +1,24 @@
import env from '#start/env'
import app from '@adonisjs/core/services/app'
/** The coordinator (REBUILD.md, part 1). */
export default {
/** Articles per range. */
rangeArticles: env.get('RANGE_ARTICLES', 2_000_000),
/** Articles per XOVER chunk (spool files, stream frames). */
chunkArticles: env.get('CHUNK_ARTICLES', 50_000),
/** A leased range without a heartbeat for this long goes back to the queue. */
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
/** NNTP connections the coordinator fetches headers with. */
nntpConnections: env.get('NNTP_CONNECTIONS', 4),
/** A reply with no data for this long fails (old ranges answer slowly). */
nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 10 * 60 * 1000),
spool: {
dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
/** Ranges kept fetched ahead, per active lease. */
aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 2),
/** At least this many ranges ahead, so the first leases start at once. */
minAhead: env.get('SPOOL_MIN_AHEAD', 2),
maxBytes: env.get('SPOOL_MAX_GB', 10) * 1e9,
},
}