Daan MeijerandClaude Opus 5.5
39480c486f
The coordinator (38) and the prober (8) had a fixed split; with history paused the coordinator's stood idle while the prober's were its limit. nntp_shares: each process syncs every 15 s (priority, minimum, what it could use, what it has open) and gets what it may open: minimums first, then by priority what each wants, growing only into connections the others closed or gave up. The pool's size and the prober's lanes follow the grant; the coordinator doubles its want while work waits for a connection, else asks for its peak in use and a margin; the prober wants its maximum unless the queue is dry. Claims scale with the lanes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
201 lines
6.4 KiB
YAML
201 lines
6.4 KiB
YAML
# Voyager 2 on homelab (deploy/README.md). deploy.sh copies this file to the
|
||
# deployment root (/mnt/bulk/usenet-rebuild), next to src/ (the code at the
|
||
# deployed commit), .env (credentials, tuning), pg/ and spool/; run
|
||
# `docker compose` there. Nothing is shared with voyager 1's compose project.
|
||
name: usenet-rebuild
|
||
|
||
x-app: &app
|
||
image: usenet-rebuild:latest
|
||
build:
|
||
context: ./src
|
||
target: app
|
||
restart: unless-stopped
|
||
env_file: .env
|
||
environment: &app-env
|
||
NODE_ENV: production
|
||
HOST: 0.0.0.0
|
||
PORT: 3334
|
||
DB_HOST: postgres
|
||
DB_PORT: 5432
|
||
APP_URL: http://web:3334
|
||
SPOOL_DIR: /spool
|
||
# Stored XOVER chunks of the ranges marked `capture` (the grouping sweeps' corpus, HDD; app/services/corpus.ts).
|
||
CORPUS_DIR: /corpus
|
||
volumes:
|
||
- ./spool:/spool
|
||
- ./corpus:/corpus
|
||
stop_grace_period: 60s
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
|
||
services:
|
||
postgres:
|
||
image: postgres:17
|
||
restart: unless-stopped
|
||
environment:
|
||
POSTGRES_USER: ${DB_USER}
|
||
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
||
POSTGRES_DB: ${DB_DATABASE}
|
||
volumes:
|
||
# Chunks (write-once payloads) on the HDD; the catalog in tablespace
|
||
# nvme, the database's default_tablespace (LOG.md 2026-10-09 (14)).
|
||
- ./pg:/var/lib/postgresql/data
|
||
- ${PG_NVME_DIR:-/var/lib/usenet-rebuild-catalog}:/var/lib/postgresql/nvme
|
||
# The WAL on the NVMe too: data/pg_wal is a symlink to /var/lib/postgresql/wal. On the HDD every commit
|
||
# waited on it (WALWrite, 56–70 ms a write with the HDD 100% busy; LOG.md 2026-10-13 (43)).
|
||
- ${PG_WAL_DIR:-/var/lib/usenet-rebuild-wal}:/var/lib/postgresql/wal
|
||
# synchronous_commit = off is set with ALTER SYSTEM (postgresql.auto.conf in the data dir).
|
||
# Memory for 62 GiB of RAM shared with other services (LOG.md 2026-10-16 (69)): the catalog's hot part in
|
||
# shared_buffers, the planner told the OS caches the rest; random_page_cost for the NVMe (the HDD tablespace
|
||
# pg_default keeps 4: ALTER TABLESPACE … SET (random_page_cost = 4)).
|
||
command:
|
||
- postgres
|
||
- -c
|
||
- shared_preload_libraries=pg_stat_statements
|
||
- -c
|
||
- shared_buffers=${PG_SHARED_BUFFERS:-8GB}
|
||
- -c
|
||
- effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-32GB}
|
||
- -c
|
||
- random_page_cost=1.1
|
||
- -c
|
||
- work_mem=16MB
|
||
- -c
|
||
- maintenance_work_mem=1GB
|
||
- -c
|
||
- max_wal_size=8GB
|
||
- -c
|
||
- checkpoint_timeout=15min
|
||
- -c
|
||
- wal_buffers=64MB
|
||
# Parallel hash joins' shared memory: 1g failed with "could not resize shared memory segment" (LOG.md 2026-10-16 (61)).
|
||
shm_size: 2g
|
||
stop_grace_period: 120s
|
||
healthcheck:
|
||
test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
||
interval: 5s
|
||
retries: 30
|
||
|
||
migrate:
|
||
<<: *app
|
||
restart: 'no'
|
||
command: ['node', 'ace', 'migration:run', '--force']
|
||
volumes: []
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
|
||
# Catalog API, Newznab and the web interface.
|
||
web:
|
||
<<: *app
|
||
command: ['node', 'bin/server.js']
|
||
environment:
|
||
<<: *app-env
|
||
HTTP_ROUTES: public
|
||
ports:
|
||
- '3334:3334'
|
||
|
||
# The edge workers' API (leases, streams, batches), apart from the web
|
||
# interface: streams hold a connection up to an hour, batches of up to
|
||
# 256 MB are parsed in-process. Only the compose network reaches it.
|
||
node-api:
|
||
<<: *app
|
||
command: ['node', 'bin/server.js']
|
||
environment:
|
||
<<: *app-env
|
||
HTTP_ROUTES: node
|
||
PORT: 3335
|
||
|
||
# Prefetching (all NNTP connections), planning, the tail, merging, settling.
|
||
coordinator:
|
||
<<: *app
|
||
command: ['node', 'ace', 'coordinator:run']
|
||
environment:
|
||
<<: *app-env
|
||
NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-38}
|
||
|
||
# Phase 2's prober: reads the probe queue's articles over connections of its
|
||
# own. The account has 50 (NNTP_ACCOUNT_CONNECTIONS): 5 reserved (the
|
||
# coordinator's plain one, the spotter's and the spot inspector's 1 each, 2
|
||
# to spare: a hedged request closes its connection and opens another; with
|
||
# 10 the spot inspector got 502 Too many connections), the other 45 shared
|
||
# by the coordinator (NNTP_MIN_CONNECTIONS–NNTP_CONNECTIONS) and the prober
|
||
# (PROBE_MIN_CONNECTIONS–PROBE_CONNECTIONS), headers first
|
||
# (app/services/nntp_shares.ts).
|
||
prober:
|
||
<<: *app
|
||
command: ['node', 'ace', 'probe:run']
|
||
volumes: []
|
||
environment:
|
||
<<: *app-env
|
||
PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-8}
|
||
|
||
# The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP.
|
||
merger:
|
||
<<: *app
|
||
command: ['node', 'ace', 'releases:merge']
|
||
volumes: []
|
||
|
||
# The junk judge: releases judged hourly by what their probes showed (dead, fake, repeated, locked, unusable);
|
||
# verdicts hide them from search. No NNTP.
|
||
judge:
|
||
<<: *app
|
||
command: ['node', 'ace', 'releases:judge']
|
||
volumes: []
|
||
|
||
# Spotnet: the spotter reads free.pt's spots (verified, with the passwords
|
||
# they name) into the spots table; the spot inspector reads each spot's NZB,
|
||
# lists its files and links the releases they match. One connection each.
|
||
spotter:
|
||
<<: *app
|
||
command: ['node', 'ace', 'spots:read']
|
||
volumes: []
|
||
|
||
spot-inspector:
|
||
<<: *app
|
||
command: ['node', 'ace', 'spots:inspect']
|
||
volumes: []
|
||
|
||
# The Rust worker; it has no NNTP account and talks to node-api only.
|
||
# Stopped before node-api (a worker holding a stream keeps the server from
|
||
# exiting).
|
||
edge:
|
||
image: usenet-rebuild-edge:latest
|
||
build:
|
||
context: ./src
|
||
target: edge
|
||
restart: unless-stopped
|
||
command: ['--threads', '${EDGE_THREADS:-8}']
|
||
environment:
|
||
COORDINATOR_URL: http://node-api:3335
|
||
NODE_TOKEN: ${NODE_TOKEN}
|
||
stop_grace_period: 60s
|
||
depends_on:
|
||
node-api:
|
||
condition: service_started
|
||
|
||
# The TypeScript worker, for comparing: docker compose --profile edge-ts up -d edge-ts
|
||
edge-ts:
|
||
<<: *app
|
||
profiles: [edge-ts]
|
||
command: ['node', 'ace', 'edge:node', '--processes', '${EDGE_THREADS:-8}']
|
||
environment:
|
||
<<: *app-env
|
||
COORDINATOR_URL: http://node-api:3335
|
||
depends_on:
|
||
node-api:
|
||
condition: service_started
|
||
|
||
# One-off ace commands: docker compose run --rm ace releases:duplicates
|
||
ace:
|
||
<<: *app
|
||
profiles: [tools]
|
||
restart: 'no'
|
||
entrypoint: ['node', 'ace']
|
||
command: ['list']
|
||
|
||
networks:
|
||
default:
|
||
name: usenet-rebuild
|