From 48f4e982f9b000f91a05147a828a237a02f1481f Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 11 Oct 2026 19:49:56 +0200 Subject: [PATCH] Rebuild: the edge workers' API in its own service The node API (leases, hour-long streams, batches of up to 256 MB JSON parsed in-process) shared the web process with the catalog API, Newznab and the web interface; voyager 1 had to make the same split on 2026-10-06. HTTP_ROUTES picks a process's routes (node, public, all by default): compose service node-api (same image, port 3335 on the compose network) serves the workers, web the rest. The workers' COORDINATOR_URL points at node-api; deploy.sh stops edge before recreating node-api. Co-Authored-By: Claude Opus 5.5 --- deploy/README.md | 7 ++-- deploy/compose.yaml | 30 ++++++++++---- deploy/deploy.sh | 6 +-- start/env.ts | 1 + start/routes.ts | 95 +++++++++++++++++++++++++++------------------ 5 files changed, 87 insertions(+), 52 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 7ccac07..da20f36 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -15,7 +15,8 @@ homelab with the deploy key, whose forced command |---|---|---| | `postgres` | postgres:17 | the catalog (tablespace `nvme`) and the chunks (HDD) | | `migrate` | `usenet-rebuild` (target `app`) | `migration:run`, before every deploy | -| `web` | `usenet-rebuild` | node API, catalog API, Newznab, web interface; port 3334 | +| `web` | `usenet-rebuild` | catalog API, Newznab, web interface; port 3334 (`HTTP_ROUTES=public`) | +| `node-api` | `usenet-rebuild` | the edge workers' API (leases, streams, batches); port 3335 on the compose network only (`HTTP_ROUTES=node`) | | `coordinator` | `usenet-rebuild` | prefetching (all NNTP connections), planning, the tail, merging, settling | | `prober` | `usenet-rebuild` | phase 2: reads the probe queue's articles over `PROBE_CONNECTIONS` (6) connections, stores what they show | | `edge` | `usenet-rebuild-edge` (target `edge`) | the Rust worker, `--threads $EDGE_THREADS` (8) | @@ -60,8 +61,8 @@ docker compose up -d --no-deps edge coordinator - Recreate app services only, with `--no-deps`; Postgres only on purpose, with the worker and coordinator stopped first. -- Stop `edge` before `web`: a worker holding a stream keeps the server from - exiting. +- Stop `edge` before `node-api`: a worker holding a stream keeps the server + from exiting. - The NNTP account is shared with voyager 1 and downloads: `NNTP_CONNECTIONS` (40) plus one plain connection, plus the prober's `PROBE_CONNECTIONS` (6), out of 50. diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 69d7a78..e0dd616 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -66,14 +66,27 @@ services: postgres: condition: service_healthy - # Node API for the edge workers (leases, streams, batches), catalog API, - # Newznab and the web interface. + # Catalog API, Newznab and the web interface. web: <<: *app command: ['node', 'bin/server.js'] + environment: + <<: *app-env + HTTP_ROUTES: public ports: - '3334:3334' + # The edge workers' API (leases, streams, batches), apart from the web + # interface: streams hold a connection up to an hour, batches of up to + # 256 MB are parsed in-process. Only the compose network reaches it. + node-api: + <<: *app + command: ['node', 'bin/server.js'] + environment: + <<: *app-env + HTTP_ROUTES: node + PORT: 3335 + # Prefetching (all NNTP connections), planning, the tail, merging, settling. coordinator: <<: *app @@ -92,8 +105,9 @@ services: <<: *app-env PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-6} - # The Rust worker; it has no NNTP account and talks to web only. Stopped - # before web (a worker holding a stream keeps the server from exiting). + # The Rust worker; it has no NNTP account and talks to node-api only. + # Stopped before node-api (a worker holding a stream keeps the server from + # exiting). edge: image: usenet-rebuild-edge:latest build: @@ -102,11 +116,11 @@ services: restart: unless-stopped command: ['--threads', '${EDGE_THREADS:-8}'] environment: - COORDINATOR_URL: http://web:3334 + COORDINATOR_URL: http://node-api:3335 NODE_TOKEN: ${NODE_TOKEN} stop_grace_period: 60s depends_on: - web: + node-api: condition: service_started # The TypeScript worker, for comparing: docker compose --profile edge-ts up -d edge-ts @@ -116,9 +130,9 @@ services: command: ['node', 'ace', 'edge:node', '--processes', '${EDGE_THREADS:-8}'] environment: <<: *app-env - COORDINATOR_URL: http://web:3334 + COORDINATOR_URL: http://node-api:3335 depends_on: - web: + node-api: condition: service_started # One-off ace commands: docker compose run --rm ace releases:duplicates diff --git a/deploy/deploy.sh b/deploy/deploy.sh index e5b7145..c680f82 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -42,13 +42,13 @@ docker compose build migrate edge echo "== running migrations" docker compose run --rm --no-deps migrate # Only the app services, never postgres. The worker first: one holding a -# stream keeps the web server from exiting. +# stream keeps the node API from exiting. echo "== recreating app services" docker compose stop edge -docker compose up -d --no-deps web coordinator prober +docker compose up -d --no-deps node-api web coordinator prober docker compose up -d --no-deps edge echo "$SHA $(date -Is)" >> "$ROOT/deployed.log" # The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor). docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null -docker compose ps --format '{{.Service}} {{.Status}}' web coordinator prober edge +docker compose ps --format '{{.Service}} {{.Status}}' node-api web coordinator prober edge echo "== deployed $SHA" diff --git a/start/env.ts b/start/env.ts index 86bab59..98c8104 100644 --- a/start/env.ts +++ b/start/env.ts @@ -65,5 +65,6 @@ export default await Env.create(new URL('../', import.meta.url), { SPOOL_MAX_GB: Env.schema.number.optional(), PREFETCH_PER_RANGE: Env.schema.number.optional(), PROBE_CONNECTIONS: Env.schema.number.optional(), + HTTP_ROUTES: Env.schema.enum.optional(['all', 'node', 'public'] as const), PREFETCH_PER_GROUP: Env.schema.number.optional(), }) diff --git a/start/routes.ts b/start/routes.ts index 7183fb2..682d47d 100644 --- a/start/routes.ts +++ b/start/routes.ts @@ -11,49 +11,68 @@ import { middleware } from '#start/kernel' import router from '@adonisjs/core/services/router' import { controllers } from '#generated/controllers' -// The web interface is public/index.html (static middleware). -router.get('/health', () => ({ ok: true })) -// Newznab API for Prowlarr, Sonarr, Radarr (from voyager 1). -router.get('/api', [controllers.Newznab, 'handle']) +import env from '#start/env' + +/** + * Which routes this process serves (HTTP_ROUTES): `node`, the edge workers' + * API only (its own service: hour-long streams and batches of up to 256 MB + * parsed in-process don't hold up the web interface; LOG.md 2026-10-11 (3)); + * `public`, everything else; `all` (the default, for development and tests). + */ +const routes = env.get('HTTP_ROUTES', 'all') +const publicRoutes = routes !== 'node' +const nodeRoutes = routes !== 'public' + +router.get('/health', () => ({ ok: true, routes })) + +if (publicRoutes) { + // The web interface is public/index.html (static middleware). + // Newznab API for Prowlarr, Sonarr, Radarr (from voyager 1). + router.get('/api', [controllers.Newznab, 'handle']) +} router .group(() => { - router - .group(() => { - router.post('signup', [controllers.NewAccount, 'store']) - router.post('login', [controllers.AccessTokens, 'store']) - }) - .prefix('auth') - .as('auth') + if (publicRoutes) { + router + .group(() => { + router.post('signup', [controllers.NewAccount, 'store']) + router.post('login', [controllers.AccessTokens, 'store']) + }) + .prefix('auth') + .as('auth') - router - .group(() => { - router.get('profile', [controllers.Profile, 'show']) - router.post('logout', [controllers.AccessTokens, 'destroy']) - }) - .prefix('account') - .as('profile') - .use(middleware.auth()) + router + .group(() => { + router.get('profile', [controllers.Profile, 'show']) + router.post('logout', [controllers.AccessTokens, 'destroy']) + }) + .prefix('account') + .as('profile') + .use(middleware.auth()) - // The catalog (REBUILD.md, part 9): open, as voyager 1's (the web interface uses it); Newznab has its key. - router.get('releases', [controllers.Releases, 'index']) - router.get('releases/:id', [controllers.Releases, 'show']).where('id', router.matchers.number()) - router.get('releases/:id/nzb', [controllers.Releases, 'nzb']).where('id', router.matchers.number()) - router.get('stats', [controllers.Releases, 'stats']) + // The catalog (REBUILD.md, part 9): open, as voyager 1's (the web interface uses it); Newznab has its key. + router.get('releases', [controllers.Releases, 'index']) + router.get('releases/:id', [controllers.Releases, 'show']).where('id', router.matchers.number()) + router.get('releases/:id/nzb', [controllers.Releases, 'nzb']).where('id', router.matchers.number()) + router.get('stats', [controllers.Releases, 'stats']) + } - // Edge workers (REBUILD.md, parts 1–2). - router - .group(() => { - router.post('lease', [controllers.Node, 'lease']) - router.get('ranges/:id/stream', [controllers.Node, 'stream']) - router.post('ranges/:id/batches', [controllers.Node, 'batch']) - router.post('ranges/:id/heartbeat', [controllers.Node, 'heartbeat']) - router.post('ranges/:id/finish', [controllers.Node, 'finish']) - router.post('ranges/:id/release', [controllers.Node, 'release']) - router.post('ranges/:id/corrupt', [controllers.Node, 'corrupt']) - }) - .prefix('node') - .as('node') - .use(middleware.auth({ guards: ['node'] })) + if (nodeRoutes) { + // Edge workers (REBUILD.md, parts 1–2). + router + .group(() => { + router.post('lease', [controllers.Node, 'lease']) + router.get('ranges/:id/stream', [controllers.Node, 'stream']) + router.post('ranges/:id/batches', [controllers.Node, 'batch']) + router.post('ranges/:id/heartbeat', [controllers.Node, 'heartbeat']) + router.post('ranges/:id/finish', [controllers.Node, 'finish']) + router.post('ranges/:id/release', [controllers.Node, 'release']) + router.post('ranges/:id/corrupt', [controllers.Node, 'corrupt']) + }) + .prefix('node') + .as('node') + .use(middleware.auth({ guards: ['node'] })) + } }) .prefix('/api/v1')