diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e680f30 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +node_modules +build +tmp +.env +.git +tools/edge-bench-rs/target diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 62f6484..6c8ad7b 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -1,9 +1,12 @@ -# Tests every push; a push to master that passes is deployed to homelab. +# Tests every push; a push to rebuild (voyager 2) that passes is deployed to +# homelab. # # Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only # run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys), -# which fetches the commit from this repo, builds the image, runs the -# migrations and recreates the app services (never postgres or redis). +# which passes "deploy-rebuild " on to ~/bin/deploy-usenet-rebuild +# (deploy/deploy.sh): it fetches the commit from this repo, builds the +# images, runs the migrations and recreates the app services (never +# postgres). See deploy/README.md. name: Test and deploy on: @@ -65,7 +68,7 @@ jobs: deploy: needs: test - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/rebuild' runs-on: ubuntu-latest steps: - name: Deploy to homelab @@ -78,4 +81,4 @@ jobs: chmod 600 ~/.ssh/deploy_key printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \ - daan@10.206.0.175 "deploy ${{ github.sha }}" + daan@10.206.0.175 "deploy-rebuild ${{ github.sha }}" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7448349 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# syntax=docker/dockerfile:1 +# +# Voyager 2's images (deploy/compose.yaml): +# --target app the AdonisJS app: web server, coordinator, migrations, ace commands +# --target edge the Rust edge worker (tools/edge-bench-rs, bin edge-node) + +# Node is pinned as in CI (.nvmrc): on 24.21, jsonschema 1.5.0 (used by Ace to +# validate commands) throws "Invalid URL" on $ref, so no ace command starts. +ARG NODE_VERSION=24.16 + +# The app, compiled, with production dependencies only. +FROM node:${NODE_VERSION}-bookworm AS app-build +WORKDIR /src +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +RUN node ace build \ + && cd build \ + && npm ci --omit=dev + +FROM node:${NODE_VERSION}-bookworm-slim AS app +ENV NODE_ENV=production +WORKDIR /app +COPY --from=app-build /src/build ./ +USER node +# Every service picks its own command, e.g. `node ace coordinator:run`. +CMD ["node", "ace", "list"] + +# The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed. +FROM rust:1-bookworm AS edge-build +WORKDIR /src +COPY tools/edge-bench-rs ./ +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/src/target \ + cargo build --release --locked --bin edge-node \ + && cp target/release/edge-node /usr/local/bin/edge-node + +FROM debian:bookworm-slim AS edge +COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node +USER 1000:1000 +ENTRYPOINT ["/usr/local/bin/edge-node"] diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..6b8cbf6 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,67 @@ +# Deploying voyager 2 + +Voyager 2 (branch `rebuild`) runs on homelab as the compose project +`usenet-rebuild`, separate from voyager 1's (`usenet-indexer`, branch +`master`): its own Postgres, network, images and directory. + +A push to `rebuild` that passes CI is deployed: the deploy job SSHes to +homelab with the deploy key, whose forced command +(`~/bin/deploy-usenet-indexer`) passes `deploy-rebuild ` on to +`~/bin/deploy-usenet-rebuild` (this directory's `deploy.sh`). + +## What runs + +| Service | Image | Does | +|---|---|---| +| `postgres` | postgres:17 | the catalog (tablespace `nvme`) and the chunks (HDD) | +| `migrate` | `usenet-rebuild` (target `app`) | `migration:run`, before every deploy | +| `web` | `usenet-rebuild` | node API, catalog API, Newznab, web interface; port 3334 | +| `coordinator` | `usenet-rebuild` | prefetching (all NNTP connections), planning, the tail, merging, settling | +| `edge` | `usenet-rebuild-edge` (target `edge`) | the Rust worker, `--threads $EDGE_THREADS` (8) | +| `edge-ts` | `usenet-rebuild` | the TypeScript worker, profile `edge-ts`, not started | +| `ace` | `usenet-rebuild` | one-off ace commands, profile `tools` | + +## On homelab + +`/mnt/bulk/usenet-rebuild` (`/home` is nearly full): + +| Path | | +|---|---| +| `compose.yaml` | copied from `src/deploy/compose.yaml` on every deploy | +| `.env` | credentials and tuning, mode 600; not in git (below) | +| `src/` | the code at the deployed commit (the build context) | +| `pg/` | Postgres data (HDD) | +| `spool/` | prefetched XOVER chunks | +| `deployed.log` | one line per deploy: sha, time (local) | +| `.cache/repo.git` | bare clone the deploys fetch into | +| `app/` | an older copy of the code with dev dependencies, for `tools/*.ts` (`r.sh`); not what runs | + +The catalog's tablespace is `/var/lib/usenet-rebuild-catalog` on the NVMe +(`PG_NVME_DIR`). Keep 10 GB free on the root filesystem. + +`.env` holds `APP_KEY`, `DB_USER`, `DB_PASSWORD`, `DB_DATABASE`, the `NNTP_*` +account, `NODE_TOKEN` (the edge worker's token, from `nodes:create`), +`CHUNK_TABLESPACE=pg_default`, `TZ`, `LOG_LEVEL`, `SESSION_DRIVER` and +optional tuning (`NNTP_CONNECTIONS`, `EDGE_THREADS`, `PG_SHARED_BUFFERS`, +anything in `start/env.ts`). Hosts, ports and paths are set in compose.yaml. + +## By hand + +```sh +cd /mnt/bulk/usenet-rebuild +~/bin/deploy-usenet-rebuild # what CI runs +docker compose ps +docker compose logs -f --since 5m coordinator +docker compose run --rm ace releases:duplicates +docker compose stop edge coordinator # pause indexing (the worker first) +docker compose up -d --no-deps edge coordinator +``` + +- Recreate app services only, with `--no-deps`; Postgres only on purpose, + with the worker and coordinator stopped first. +- Stop `edge` before `web`: a worker holding a stream keeps the server from + exiting. +- The NNTP account is shared with voyager 1 and downloads: `NNTP_CONNECTIONS` + (40) plus one plain connection, out of 50. +- `deploy.sh` doesn't update itself: after changing it, install it again + (`install -m 755 src/deploy/deploy.sh ~/bin/deploy-usenet-rebuild`). diff --git a/deploy/compose.yaml b/deploy/compose.yaml new file mode 100644 index 0000000..3e31613 --- /dev/null +++ b/deploy/compose.yaml @@ -0,0 +1,124 @@ +# Voyager 2 on homelab (deploy/README.md). deploy.sh copies this file to the +# deployment root (/mnt/bulk/usenet-rebuild), next to src/ (the code at the +# deployed commit), .env (credentials, tuning), pg/ and spool/; run +# `docker compose` there. Nothing is shared with voyager 1's compose project. +name: usenet-rebuild + +x-app: &app + image: usenet-rebuild:latest + build: + context: ./src + target: app + restart: unless-stopped + env_file: .env + environment: &app-env + NODE_ENV: production + HOST: 0.0.0.0 + PORT: 3334 + DB_HOST: postgres + DB_PORT: 5432 + APP_URL: http://web:3334 + SPOOL_DIR: /spool + volumes: + - ./spool:/spool + stop_grace_period: 60s + depends_on: + migrate: + condition: service_completed_successfully + +services: + postgres: + image: postgres:17 + restart: unless-stopped + environment: + POSTGRES_USER: ${DB_USER} + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_DATABASE} + volumes: + # Chunks (write-once payloads) on the HDD; the catalog in tablespace + # nvme, the database's default_tablespace (LOG.md 2026-10-09 (14)). + - ./pg:/var/lib/postgresql/data + - ${PG_NVME_DIR:-/var/lib/usenet-rebuild-catalog}:/var/lib/postgresql/nvme + # synchronous_commit = off is set with ALTER SYSTEM (postgresql.auto.conf in the data dir). + command: + - postgres + - -c + - shared_preload_libraries=pg_stat_statements + - -c + - shared_buffers=${PG_SHARED_BUFFERS:-2GB} + - -c + - max_wal_size=8GB + - -c + - checkpoint_timeout=15min + shm_size: 1g + stop_grace_period: 120s + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"'] + interval: 5s + retries: 30 + + migrate: + <<: *app + restart: 'no' + command: ['node', 'ace', 'migration:run', '--force'] + volumes: [] + depends_on: + postgres: + condition: service_healthy + + # Node API for the edge workers (leases, streams, batches), catalog API, + # Newznab and the web interface. + web: + <<: *app + command: ['node', 'bin/server.js'] + ports: + - '3334:3334' + + # Prefetching (all NNTP connections), planning, the tail, merging, settling. + coordinator: + <<: *app + command: ['node', 'ace', 'coordinator:run'] + environment: + <<: *app-env + NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-40} + + # The Rust worker; it has no NNTP account and talks to web only. Stopped + # before web (a worker holding a stream keeps the server from exiting). + edge: + image: usenet-rebuild-edge:latest + build: + context: ./src + target: edge + restart: unless-stopped + command: ['--threads', '${EDGE_THREADS:-8}'] + environment: + COORDINATOR_URL: http://web:3334 + NODE_TOKEN: ${NODE_TOKEN} + stop_grace_period: 60s + depends_on: + web: + condition: service_started + + # The TypeScript worker, for comparing: docker compose --profile edge-ts up -d edge-ts + edge-ts: + <<: *app + profiles: [edge-ts] + command: ['node', 'ace', 'edge:node', '--processes', '${EDGE_THREADS:-8}'] + environment: + <<: *app-env + COORDINATOR_URL: http://web:3334 + depends_on: + web: + condition: service_started + + # One-off ace commands: docker compose run --rm ace releases:duplicates + ace: + <<: *app + profiles: [tools] + restart: 'no' + entrypoint: ['node', 'ace'] + command: ['list'] + +networks: + default: + name: usenet-rebuild diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 0000000..7e37f1b --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Deploys voyager 2 (branch rebuild) on homelab at one commit. Installed as +# ~/bin/deploy-usenet-rebuild (by hand: it doesn't update itself). Gitea CI +# runs it through the forced SSH command of the deploy key, as +# "deploy-rebuild " (~/bin/deploy-usenet-indexer passes that on); by +# hand: deploy-usenet-rebuild . The code comes from the Gitea repo, not +# from the CI job. +set -eu + +REPO=https://gitea.exp.sforum.nl/daan/usenet-indexer.git +BRANCH=rebuild +ROOT=/mnt/bulk/usenet-rebuild +CACHE="$ROOT/.cache" +GIT="$CACHE/repo.git" + +SHA="${1:-}" +case "$SHA" in + *[!0-9a-f]* | "") echo "usage: deploy-usenet-rebuild <40-character commit sha>" >&2; exit 2 ;; +esac +[ "${#SHA}" -eq 40 ] || { echo "usage: deploy-usenet-rebuild <40-character commit sha>" >&2; exit 2; } + +mkdir -p "$CACHE" +# One deploy at a time; a second one waits. +exec 9>"$CACHE/deploy.lock" +flock 9 + +echo "== fetching $SHA" +[ -d "$GIT" ] || git clone --quiet --bare "$REPO" "$GIT" +git -C "$GIT" fetch --quiet --prune "$REPO" '+refs/heads/*:refs/heads/*' +git -C "$GIT" cat-file -e "$SHA^{commit}" +git -C "$GIT" merge-base --is-ancestor "$SHA" "refs/heads/$BRANCH" || { echo "$SHA is not on $BRANCH" >&2; exit 2; } + +STAGE=$(mktemp -d) +trap 'rm -rf "$STAGE"' EXIT +git -C "$GIT" archive "$SHA" | tar -x -C "$STAGE" +rsync -a --delete "$STAGE"/ "$ROOT/src/" +install -m 644 "$ROOT/src/deploy/compose.yaml" "$ROOT/compose.yaml" + +cd "$ROOT" +echo "== building images" +docker compose build migrate edge +echo "== running migrations" +docker compose run --rm --no-deps migrate +# Only the app services, never postgres. The worker first: one holding a +# stream keeps the web server from exiting. +echo "== recreating app services" +docker compose stop edge +docker compose up -d --no-deps web coordinator +docker compose up -d --no-deps edge +echo "$SHA $(date -Is)" >> "$ROOT/deployed.log" +docker compose ps --format '{{.Service}} {{.Status}}' web coordinator edge +echo "== deployed $SHA"