Daan MeijerandClaude Opus 5.5
8bd6d6aa5f
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
135 lines
3.7 KiB
YAML
135 lines
3.7 KiB
YAML
# Voyager 2 on homelab (deploy/README.md). deploy.sh copies this file to the
|
|
# deployment root (/mnt/bulk/usenet-rebuild), next to src/ (the code at the
|
|
# deployed commit), .env (credentials, tuning), pg/ and spool/; run
|
|
# `docker compose` there. Nothing is shared with voyager 1's compose project.
|
|
name: usenet-rebuild
|
|
|
|
x-app: &app
|
|
image: usenet-rebuild:latest
|
|
build:
|
|
context: ./src
|
|
target: app
|
|
restart: unless-stopped
|
|
env_file: .env
|
|
environment: &app-env
|
|
NODE_ENV: production
|
|
HOST: 0.0.0.0
|
|
PORT: 3334
|
|
DB_HOST: postgres
|
|
DB_PORT: 5432
|
|
APP_URL: http://web:3334
|
|
SPOOL_DIR: /spool
|
|
volumes:
|
|
- ./spool:/spool
|
|
stop_grace_period: 60s
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:17
|
|
restart: unless-stopped
|
|
environment:
|
|
POSTGRES_USER: ${DB_USER}
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
|
POSTGRES_DB: ${DB_DATABASE}
|
|
volumes:
|
|
# Chunks (write-once payloads) on the HDD; the catalog in tablespace
|
|
# nvme, the database's default_tablespace (LOG.md 2026-10-09 (14)).
|
|
- ./pg:/var/lib/postgresql/data
|
|
- ${PG_NVME_DIR:-/var/lib/usenet-rebuild-catalog}:/var/lib/postgresql/nvme
|
|
# synchronous_commit = off is set with ALTER SYSTEM (postgresql.auto.conf in the data dir).
|
|
command:
|
|
- postgres
|
|
- -c
|
|
- shared_preload_libraries=pg_stat_statements
|
|
- -c
|
|
- shared_buffers=${PG_SHARED_BUFFERS:-2GB}
|
|
- -c
|
|
- max_wal_size=8GB
|
|
- -c
|
|
- checkpoint_timeout=15min
|
|
shm_size: 1g
|
|
stop_grace_period: 120s
|
|
healthcheck:
|
|
test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
|
interval: 5s
|
|
retries: 30
|
|
|
|
migrate:
|
|
<<: *app
|
|
restart: 'no'
|
|
command: ['node', 'ace', 'migration:run', '--force']
|
|
volumes: []
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
|
|
# Node API for the edge workers (leases, streams, batches), catalog API,
|
|
# Newznab and the web interface.
|
|
web:
|
|
<<: *app
|
|
command: ['node', 'bin/server.js']
|
|
ports:
|
|
- '3334:3334'
|
|
|
|
# Prefetching (all NNTP connections), planning, the tail, merging, settling.
|
|
coordinator:
|
|
<<: *app
|
|
command: ['node', 'ace', 'coordinator:run']
|
|
environment:
|
|
<<: *app-env
|
|
NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-40}
|
|
|
|
# Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS
|
|
# connections of its own (next to the coordinator's 40 + 1; the account has 50).
|
|
prober:
|
|
<<: *app
|
|
command: ['node', 'ace', 'probe:run']
|
|
volumes: []
|
|
environment:
|
|
<<: *app-env
|
|
PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-6}
|
|
|
|
# The Rust worker; it has no NNTP account and talks to web only. Stopped
|
|
# before web (a worker holding a stream keeps the server from exiting).
|
|
edge:
|
|
image: usenet-rebuild-edge:latest
|
|
build:
|
|
context: ./src
|
|
target: edge
|
|
restart: unless-stopped
|
|
command: ['--threads', '${EDGE_THREADS:-8}']
|
|
environment:
|
|
COORDINATOR_URL: http://web:3334
|
|
NODE_TOKEN: ${NODE_TOKEN}
|
|
stop_grace_period: 60s
|
|
depends_on:
|
|
web:
|
|
condition: service_started
|
|
|
|
# The TypeScript worker, for comparing: docker compose --profile edge-ts up -d edge-ts
|
|
edge-ts:
|
|
<<: *app
|
|
profiles: [edge-ts]
|
|
command: ['node', 'ace', 'edge:node', '--processes', '${EDGE_THREADS:-8}']
|
|
environment:
|
|
<<: *app-env
|
|
COORDINATOR_URL: http://web:3334
|
|
depends_on:
|
|
web:
|
|
condition: service_started
|
|
|
|
# One-off ace commands: docker compose run --rm ace releases:duplicates
|
|
ace:
|
|
<<: *app
|
|
profiles: [tools]
|
|
restart: 'no'
|
|
entrypoint: ['node', 'ace']
|
|
command: ['list']
|
|
|
|
networks:
|
|
default:
|
|
name: usenet-rebuild
|