Accounts with limits: a Newznab key and a tier per user, daily API and NZB counts, invites

ACCESS=accounts (config/access.ts; open stays the default and behaves as
before): Newznab and the catalog's releases routes name a user (apikey or
the web interface's token), whose tier sets searches and NZBs per UTC day
(Newznab 500/501, the catalog 429) and how far back they see (102/403).
Counted in one upsert per use, so the last one can't be taken twice;
caps, stats and refused requests aren't counted. Signup takes an invite
(SIGNUP), which may give a tier; ace access:invite, access:tier (extends a
running tier), access:suspend. The web interface signs in and shows the
day's usage and the Newznab key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.adonisjs/client/manifest.d.ts

@@ -5,10 +5,6 @@
 
 /// <reference path="../../adonisrc.ts" />
 /// <reference path="../../config/auth.ts" />
-/// <reference path="../../config/gc.ts" />
+/// <reference path="../../config/coordinator.ts" />
 /// <reference path="../../config/hash.ts" />
-/// <reference path="../../config/indexer.ts" />
 /// <reference path="../../config/logger.ts" />
-/// <reference path="../../config/nntp.ts" />
-/// <reference path="../../config/queue.ts" />
-/// <reference path="../../config/redis.ts" />

.adonisjs/client/registry/index.ts

@@ -30,6 +30,54 @@ const routes = {
     tokens: [{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],
     types: placeholder as Registry['profile.access_tokens.destroy']['types'],
   },
+  'releases.index': {
+    methods: ["GET","HEAD"],
+    pattern: '/api/v1/releases',
+    tokens: [{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],
+    types: placeholder as Registry['releases.index']['types'],
+  },
+  'releases.nzb': {
+    methods: ["GET","HEAD"],
+    pattern: '/api/v1/releases/:id/nzb',
+    tokens: [{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],
+    types: placeholder as Registry['releases.nzb']['types'],
+  },
+  'node.node.lease': {
+    methods: ["POST"],
+    pattern: '/api/v1/node/lease',
+    tokens: [{"old":"/api/v1/node/lease","type":0,"val":"api","end":""},{"old":"/api/v1/node/lease","type":0,"val":"v1","end":""},{"old":"/api/v1/node/lease","type":0,"val":"node","end":""},{"old":"/api/v1/node/lease","type":0,"val":"lease","end":""}],
+    types: placeholder as Registry['node.node.lease']['types'],
+  },
+  'node.node.stream': {
+    methods: ["GET","HEAD"],
+    pattern: '/api/v1/node/ranges/:id/stream',
+    tokens: [{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"stream","end":""}],
+    types: placeholder as Registry['node.node.stream']['types'],
+  },
+  'node.node.batch': {
+    methods: ["POST"],
+    pattern: '/api/v1/node/ranges/:id/batches',
+    tokens: [{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"batches","end":""}],
+    types: placeholder as Registry['node.node.batch']['types'],
+  },
+  'node.node.heartbeat': {
+    methods: ["POST"],
+    pattern: '/api/v1/node/ranges/:id/heartbeat',
+    tokens: [{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"heartbeat","end":""}],
+    types: placeholder as Registry['node.node.heartbeat']['types'],
+  },
+  'node.node.finish': {
+    methods: ["POST"],
+    pattern: '/api/v1/node/ranges/:id/finish',
+    tokens: [{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"finish","end":""}],
+    types: placeholder as Registry['node.node.finish']['types'],
+  },
+  'node.node.release': {
+    methods: ["POST"],
+    pattern: '/api/v1/node/ranges/:id/release',
+    tokens: [{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/release","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"release","end":""}],
+    types: placeholder as Registry['node.node.release']['types'],
+  },
 } as const satisfies Record<string, AdonisEndpoint>
 
 export { routes }

.adonisjs/client/registry/schema.d.ts

@@ -55,4 +55,100 @@ export interface Registry {
       errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['destroy']>>>
     }
   }
+  'releases.index': {
+    methods: ["GET","HEAD"]
+    pattern: '/api/v1/releases'
+    types: {
+      body: {}
+      paramsTuple: []
+      params: {}
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['index']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['index']>>>
+    }
+  }
+  'releases.nzb': {
+    methods: ["GET","HEAD"]
+    pattern: '/api/v1/releases/:id/nzb'
+    types: {
+      body: {}
+      paramsTuple: [ParamValue]
+      params: { id: ParamValue }
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['nzb']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['nzb']>>>
+    }
+  }
+  'node.node.lease': {
+    methods: ["POST"]
+    pattern: '/api/v1/node/lease'
+    types: {
+      body: {}
+      paramsTuple: []
+      params: {}
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['lease']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['lease']>>>
+    }
+  }
+  'node.node.stream': {
+    methods: ["GET","HEAD"]
+    pattern: '/api/v1/node/ranges/:id/stream'
+    types: {
+      body: {}
+      paramsTuple: [ParamValue]
+      params: { id: ParamValue }
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['stream']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['stream']>>>
+    }
+  }
+  'node.node.batch': {
+    methods: ["POST"]
+    pattern: '/api/v1/node/ranges/:id/batches'
+    types: {
+      body: {}
+      paramsTuple: [ParamValue]
+      params: { id: ParamValue }
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['batch']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['batch']>>>
+    }
+  }
+  'node.node.heartbeat': {
+    methods: ["POST"]
+    pattern: '/api/v1/node/ranges/:id/heartbeat'
+    types: {
+      body: {}
+      paramsTuple: [ParamValue]
+      params: { id: ParamValue }
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['heartbeat']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['heartbeat']>>>
+    }
+  }
+  'node.node.finish': {
+    methods: ["POST"]
+    pattern: '/api/v1/node/ranges/:id/finish'
+    types: {
+      body: {}
+      paramsTuple: [ParamValue]
+      params: { id: ParamValue }
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['finish']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['finish']>>>
+    }
+  }
+  'node.node.release': {
+    methods: ["POST"]
+    pattern: '/api/v1/node/ranges/:id/release'
+    types: {
+      body: {}
+      paramsTuple: [ParamValue]
+      params: { id: ParamValue }
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['release']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['release']>>>
+    }
+  }
 }

.adonisjs/client/registry/tree.d.ts

@@ -18,4 +18,18 @@ export interface ApiDefinition {
       destroy: typeof routes['profile.access_tokens.destroy']
     }
   }
+  releases: {
+    index: typeof routes['releases.index']
+    nzb: typeof routes['releases.nzb']
+  }
+  node: {
+    node: {
+      lease: typeof routes['node.node.lease']
+      stream: typeof routes['node.node.stream']
+      batch: typeof routes['node.node.batch']
+      heartbeat: typeof routes['node.node.heartbeat']
+      finish: typeof routes['node.node.finish']
+      release: typeof routes['node.node.release']
+    }
+  }
 }

.adonisjs/server/controllers.ts

@@ -7,6 +7,7 @@ export const controllers = {
   AccessTokens: () => import('#controllers/access_tokens_controller'),
   NewAccount: () => import('#controllers/new_account_controller'),
   Newznab: () => import('#controllers/newznab_controller'),
+  Node: () => import('#controllers/node_controller'),
   Profile: () => import('#controllers/profile_controller'),
   Releases: () => import('#controllers/releases_controller'),
 }

.adonisjs/server/routes.d.ts

@@ -4,33 +4,51 @@ type ParamValue = string | number | bigint | boolean
 
 export type ScannedRoutes = {
   ALL: {
-    'releases.index': { paramsTuple?: []; params?: {} }
-    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
-    'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
-    'releases.stats': { paramsTuple?: []; params?: {} }
+    'newznab': { paramsTuple?: []; params?: {} }
     'auth.new_account.store': { paramsTuple?: []; params?: {} }
     'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
     'profile.profile.show': { paramsTuple?: []; params?: {} }
     'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
+    'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
+    'node.node.lease': { paramsTuple?: []; params?: {} }
+    'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   GET: {
+    'newznab': { paramsTuple?: []; params?: {} }
+    'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
     'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.stats': { paramsTuple?: []; params?: {} }
-    'profile.profile.show': { paramsTuple?: []; params?: {} }
+    'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   HEAD: {
+    'newznab': { paramsTuple?: []; params?: {} }
+    'profile.profile.show': { paramsTuple?: []; params?: {} }
     'releases.index': { paramsTuple?: []; params?: {} }
     'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
     'releases.stats': { paramsTuple?: []; params?: {} }
-    'profile.profile.show': { paramsTuple?: []; params?: {} }
+    'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
   POST: {
     'auth.new_account.store': { paramsTuple?: []; params?: {} }
     'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
     'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
+    'node.node.lease': { paramsTuple?: []; params?: {} }
+    'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
   }
 }
 declare module '@adonisjs/core/types/http' {

.adonisjs/server/routes.json

@@ -1 +0,0 @@
-{"root":[{"domain":"root","pattern":"/api/v1/releases","matchers":{},"tokens":[{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],"meta":{"params":[]},"name":"releases.index","handler":{"method":"index","reference":[null,"index"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id","type":1,"val":"id","end":"","matcher":{}}],"meta":{"params":["id"]},"name":"releases.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id/nzb","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":"","matcher":{}},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],"meta":{"params":["id"]},"name":"releases.nzb","handler":{"method":"nzb","reference":[null,"nzb"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/stats","matchers":{},"tokens":[{"old":"/api/v1/stats","type":0,"val":"api","end":""},{"old":"/api/v1/stats","type":0,"val":"v1","end":""},{"old":"/api/v1/stats","type":0,"val":"stats","end":""}],"meta":{"params":[]},"name":"releases.stats","handler":{"method":"stats","reference":[null,"stats"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/signup","matchers":{},"tokens":[{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],"meta":{"params":[]},"name":"auth.new_account.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/new_account_controller')","name":"NewAccount"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/login","matchers":{},"tokens":[{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],"meta":{"params":[]},"name":"auth.access_tokens.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/profile","matchers":{},"tokens":[{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],"meta":{"params":[]},"name":"profile.profile.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/profile_controller')","name":"Profile"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/logout","matchers":{},"tokens":[{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],"meta":{"params":[]},"name":"profile.access_tokens.destroy","handler":{"method":"destroy","reference":[null,"destroy"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}}]}

.dockerignore

@@ -1,11 +1,6 @@
 node_modules
 build
 tmp
-debug
-.git
-.idea
-.vscode
 .env
-.env.*
-!.env.example
-*.log
+.git
+edge/target

.env.example

@@ -12,9 +12,19 @@ APP_URL=http://${HOST}:${PORT}
 # Session
 SESSION_DRIVER=cookie
 
+# Database (the rebuild's own)
+DB_HOST=127.0.0.1
+DB_PORT=5432
+DB_USER=usenet_indexer
+DB_PASSWORD=
+DB_DATABASE=usenet_rebuild
+
+# Access (config/access.ts): open (NEWZNAB_API_KEY or none, no limits) or accounts
+# (a Newznab key per user, tiers with daily limits; ace access:invite, access:tier, access:suspend)
+# ACCESS=open
+# SIGNUP=open|invite|closed (with accounts: invite unless set)
+
 #--------------------------------------------------------------------
 # CORS (configure allowed origins for API access)
 #--------------------------------------------------------------------
-# CORS_ORIGIN=http://localhost:5173,http://localhost:3000
-# Newznab API key (/api?apikey=...); leave empty to allow access without a key
-NEWZNAB_API_KEY=
+# CORS_ORIGIN=http://localhost:5173,http://localhost:3000

.env.test

@@ -1 +1,2 @@
 SESSION_DRIVER=memory
+DB_DATABASE=usenet_rebuild_test

.gitea/workflows/deploy.yaml

@@ -0,0 +1,104 @@
+# Tests every push; a push to rebuild (voyager 2) that passes is deployed to
+# homelab.
+#
+# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
+# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
+# which passes "deploy-rebuild <sha>" on to ~/bin/deploy-usenet-rebuild
+# (deploy/deploy.sh): it fetches the commit from this repo, builds the
+# images, runs the migrations and recreates the app services (never
+# postgres). See deploy/README.md.
+name: Test and deploy
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: indexer
+          POSTGRES_PASSWORD: indexer
+          POSTGRES_DB: usenet_indexer
+        options: >-
+          --health-cmd "pg_isready -U indexer -d usenet_indexer"
+          --health-interval 5s
+          --health-retries 20
+      redis:
+        image: redis:8
+    env:
+      TZ: UTC
+      PORT: 3333
+      HOST: 0.0.0.0
+      NODE_ENV: test
+      LOG_LEVEL: warn
+      APP_KEY: ci-only-app-key-not-a-secret-000000
+      APP_URL: http://localhost:3333
+      SESSION_DRIVER: memory
+      DB_CONNECTION: pg
+      DB_HOST: postgres
+      DB_PORT: 5432
+      DB_USER: indexer
+      DB_PASSWORD: indexer
+      DB_DATABASE: usenet_indexer
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      # Tests never connect to a news server.
+      NNTP_HOST: localhost
+      NNTP_PORT: 563
+      NNTP_USER: ci
+      NNTP_PASSWORD: ci
+      NNTP_SECURE: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Type check
+        run: npx tsc --noEmit
+      - name: Migrate
+        run: node ace migration:run --force
+      - name: Test
+        run: node ace test
+
+  # The edge worker (edge/, Rust): its tests, and the Rust and TypeScript
+  # RangeIndexers compared on the committed captures (edge/fixtures).
+  edge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Rust toolchain
+        run: |
+          command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+      - name: cargo test
+        run: cargo test --release --manifest-path edge/Cargo.toml
+      - name: Rust and TypeScript identical on the fixtures
+        run: edge/check-fixtures.sh
+
+  deploy:
+    needs: [test, edge]
+    if: github.ref == 'refs/heads/rebuild'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to homelab
+        env:
+          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+          DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
+        run: |
+          install -m 700 -d ~/.ssh
+          printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
+          ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
+            daan@10.206.0.175 "deploy-rebuild ${{ github.sha }}"

.gitignore

@@ -24,3 +24,6 @@ yarn-error.log
 
 # Platform specific
 .DS_Store
+
+# Generated by the dev tooling
+.adonisjs/server/routes.json

Dockerfile

@@ -1,13 +1,16 @@
 # syntax=docker/dockerfile:1
+#
+# Voyager 2's images (deploy/compose.yaml):
+#   --target app   the AdonisJS app: web server, coordinator, migrations, ace commands
+#   --target edge  the Rust edge worker (edge/, bin edge-node)
 
-# Node is pinned: on 24.21, jsonschema 1.5.0 (used by Ace to validate commands)
-# throws "Invalid URL" on $ref, so no ace command starts. See LOG.md.
+# Node is pinned as in CI (.nvmrc): on 24.21, jsonschema 1.5.0 (used by Ace to
+# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
 ARG NODE_VERSION=24.16
 
-# Build stage: dev dependencies, TypeScript compile, native addons.
-# The full node image ships python3/make/g++, needed to compile yencode.
-FROM node:${NODE_VERSION}-bookworm AS build
-WORKDIR /app
+# The app, compiled, with production dependencies only.
+FROM node:${NODE_VERSION}-bookworm AS app-build
+WORKDIR /src
 COPY package.json package-lock.json ./
 RUN npm ci
 COPY . .
@@ -15,15 +18,26 @@ RUN node ace build \
  && cd build \
  && npm ci --omit=dev
 
-# Runtime stage: compiled app with production dependencies only.
-FROM node:${NODE_VERSION}-bookworm-slim
+FROM node:${NODE_VERSION}-bookworm-slim AS app
 ENV NODE_ENV=production
-# bsdtar (libarchive) unpacks compressed RAR entries, e.g. NZBs posted inside RARs.
-RUN apt-get update \
- && apt-get install -y --no-install-recommends libarchive-tools \
- && rm -rf /var/lib/apt/lists/*
 WORKDIR /app
-COPY --from=build /app/build ./
+COPY --from=app-build /src/build ./
 USER node
-# Every service picks its own command, e.g. `node ace worker:fetch`.
+# Every service picks its own command, e.g. `node ace coordinator:run`.
 CMD ["node", "ace", "list"]
+
+# The Rust worker. Registry and target dir are build caches, so a rebuild only compiles what changed.
+FROM rust:1-bookworm AS edge-build
+WORKDIR /src
+COPY edge ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/src/target \
+    cargo build --release --locked --bin edge-node --bin edge-bench \
+ && cp target/release/edge-node target/release/edge-bench /usr/local/bin/
+
+FROM debian:bookworm-slim AS edge
+COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
+# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
+COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
+USER 1000:1000
+ENTRYPOINT ["/usr/local/bin/edge-node"]

REBUILD.md

@@ -0,0 +1,328 @@
+# Voyager 2: header-first indexer (concept)
+
+The concept for what runs (since 2026-10-11; it started as the spec of a
+rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
+decisions D1–D42 are superseded; its domain model still holds, and its
+features not yet here are the phase 2 backlog below.
+
+Branch `rebuild`. Phase 1 finds releases from headers and makes their NZBs.
+Everything learned from articles (names from yEnc headers, par2, archive
+listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
+never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
+
+## Status (2026-10-09)
+
+Built on this branch (a fresh AdonisJS 7 app; phase 1 code imported from
+the current app by pipeline stage): coordinator (planner, NNTP pool,
+prefetch spool, leases, rule sets, fragment merger, settling), the node API
+(lease, framed stream, batches, finish/release/heartbeat), the edge worker
+(`edge:node`, `--processes`), the catalog and chunk store, NZBs and a name
+search. Measured on the workstation (4 cores, 100 Mbit, Eweka):
+
+- Edge worker, one core: ~210–215 k headers/s on recent a.b.boneless,
+  ~330–340 k on a 2013 a.b.ath range; ~0.7–0.8 GB peak per process.
+  (Start: 73–103 k/s, 1.0–1.2 GB.) What's left is spread out (the
+  per-file key ~12%, GC ~10%, zlib ~10%, the scan ~6%).
+- NNTP: Eweka takes ~1.5–2 s per XOVER command; ~22–30 k lines/s per
+  connection in chunks of 100 k when the line allows; connections don't
+  slow each other.
+- Bandwidth: compressed XOVER is ~69 B per article on recent boneless
+  (100 k-article chunks of ~7 MB). The workstation's 100 Mbit line carries
+  ~140–150 k articles/s, which is where its end-to-end runs now stop (10
+  connections 90–99% busy, ~89 Mbit/s).
+- End to end at that rate (one 4-core machine): workers about half busy
+  each; server side ~17% (coordinator) + ~8% (web) + ~3% (Postgres) of a
+  core. (Before this round's fixes Postgres was at 32%: the merger's
+  neighbour query read every range per file.)
+- On homelab (12 cores, 1 Gbit; 2026-10-09, separate containers and
+  database, `/mnt/bulk/usenet-rebuild`): 20 connections fetched ~630 k
+  articles/s (~31 k per connection, ~3 s per 100 k chunk, ~250 Mbit/s at
+  ~50 B per article on older boneless) and 8 worker processes indexed all of
+  it, waiting for the spool (~366 M headers in ~12 minutes). Server side at
+  that rate: coordinator 0.4–0.5 cores (1.3–1.7 before it stopped
+  inflating replies to check them), web 0.3–0.45, Postgres (on the HDD)
+  under 0.15. Worker per core there: ~195 k headers/s (boneless 2026),
+  ~275 k (ath 2013).
+- Storage: chunks 17.9–20.3 B per segment (format v2); a release ~1.1 KB.
+- For the whole target (528 B overview lines): ~36 TB of compressed XOVER
+  to download. At 50 connections (~1–1.25 M lines/s, ~0.6–0.7 Gbit/s)
+  ~5–6 days, ~5–6 worker cores, two coordinator shards (one process
+  handles ~60 MB/s), under a core for web and Postgres. The line is the
+  limit below ~0.7 Gbit/s: at 1 Gbit, ~3.5 days at best.
+
+Where workers run: total speed is set by the account's NNTP connections
+(~30 k articles/s each), not by worker CPU, so remote workers don't make
+it faster; they take CPU off homelab (at 1.25 M/s ~7 worker cores) at the
+cost of relaying the compressed XOVER (≤ ~0.7 Gbit/s up, homelab has 1
+Gbit/s). Plan: coordinator, web, Postgres and a few workers on homelab,
+most workers on batcave, a few on poweredge-kantoor with
+`edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
+the cap is shared by the node's processes).
+
+Also built (2026-10-09): the live tail (new articles planned every 2
+minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
+fetched before history); the catalog API, Newznab (`/api`) and the web
+interface, from voyager 1 (the current build), with categories from the
+name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
+`database/data/voyager1_signatures.tsv`).
+
+Deployment (2026-10-10): `Dockerfile` (targets app and edge),
+`deploy/compose.yaml` (project usenet-rebuild on homelab), `deploy/deploy.sh`;
+a push to `rebuild` that passes CI is deployed. See `deploy/README.md`.
+
+Not built yet: re-running ranges for a retracted rule (the counts and rule
+versions are recorded), decision f, phase
+2 (inspection of the probe queue: yEnc names, encryption, archives, par2;
+passwords; spots).
+
+## Target
+
+7,000 days (all of Eweka's retention, back to 2008-08) of the 1000 largest
+`alt.binaries.*` groups. Estimated: 528 B overview lines; 435 B in their
+main group; 140 B kept segments in ~1.25 B files and ~115 M releases after
+dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
+≥ 25 B overview lines/day (~50 connections), so ~3 weeks.
+
+## Parts
+
+1. **Coordinator** (server). Plans each group's history as ranges of 2 M
+   articles, leases them to workers, tracks progress and retries. The
+   group list (owner rule) and the garbage signature table go with every
+   lease. Fetches XOVER for the workers (relay: workers have no NNTP
+   account; the gzip stream is passed on as is).
+
+   **Prefetch**: the coordinator fetches the next ranges ahead of the
+   workers, in chunks of 100 k lines and in order, into a spool on disk
+   (~106 MB per 2 M range at ~53 B per line; on the HDD, written and read
+   sequentially). It keeps about two ranges ahead per active lease, capped
+   by size (default 10 GB), and fetches with all its header connections all
+   the time, independent of how fast workers are. A lease goes to a range
+   that is already (partly) spooled, else to the next planned one (leased
+   ranges are fetched first); a worker reads its chunks from the spool,
+   waiting only for chunks still being fetched. A worker gets its
+   whole range in one streaming request (`GET /ranges/:id/stream`): the
+   spooled chunks in order, waiting for those still being fetched. It still
+   parses chunk by chunk (2 M lines are ~560 MB of text) and posts results
+   per batch, moving the range's resume point. Chunks matter on the NNTP
+   side: a compressed XOVER reply can't be resumed, old ranges answer
+   slowly, and chunks of one range can be fetched in parallel. Spooled chunks are
+   deleted when their range's results are stored, and fetched again if a
+   range is leased again later.
+   Order: oldest era first (2008–2018 holds 63% of the readable files on
+   17% of the lines), then forward.
+   One coordinator process handles ~60 MB/s of compressed XOVER (its event
+   loop was 16.5% busy at ~10 MB/s); beyond that, prefetching is split over
+   processes with `coordinator:run --shard=i/n` (ranges with id % n = i − 1,
+   each with its own NNTP_CONNECTIONS; only shard 1 plans, merges and
+   settles).
+
+2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`,
+   2026-10-09, after two rounds of work): ~200–215 k headers/s per core on
+   recent a.b.boneless, ~320–340 k on a 2013 a.b.ath range, ~0.7 GB peak
+   per process. **Rust** (`edge/`, the same RangeIndexer path
+   over the same captures; `check.ts` finds every output identical, the
+   decoded payloads included): 3.5–3.8× less CPU (boneless 2.9 vs 11.1
+   CPU-s per 2 M headers, ath 2.15 vs 7.5) and a third of the memory
+   (0.20–0.24 GB). Its gains: no GC, hand-written matching instead of the
+   regexes, mimalloc, libdeflate (half of zlib's inflate time) and SIMD line
+   scanning. At the full target (~1.25 M/s) that is ~1.5–2 worker cores
+   instead of ~6–7; the speed is still set by the NNTP connections. The
+   cost: a second implementation of lib/headers, lib/release, lib/rules and
+   the chunk format, kept identical (check.ts) for as long as both exist.
+   Findings, how to run both and how to compare their outputs:
+   `edge/README.md`. Since 2026-10-09 the Rust worker
+   (`edge-node`, same crate) is the one running on homelab: ~8× less CPU
+   per header and 20× less memory than `edge:node` in production.
+   Per lease: compressed chunks of 10–50 k lines, in order →
+   parse subjects → skip crossposts outside their owner group (of the
+   indexed groups in the article's Xref, the one with the lowest priority
+   number; see the group list) → group articles into files → files
+   into releases (`lib/release` rules) → drop garbage (random name +
+   dropped signature or tool) → set aside per-article random subjects and
+   obfuscated subjects (counted only; phase 2). Returns per batch: release
+   chunks, edge fragments, probe candidates, counters. Holds no state beyond
+   its lease.
+
+   **Probe candidates**: per release, the articles phase 2 should read, as
+   (release key, kind, message-id, file in its chunk): the first article of
+   the first archive volume (`.part01.rar`, `.rar`, `.7z.001`), of the par2
+   index (smallest `.par2`), and of a lone random-named file; for a 7z set
+   also its last volume (the listing is in the tail). At most a few per
+   release (~115 M releases → ~150–250 M articles), not one per file. Per-
+   article and obfuscated subjects get none (they need every article's body).
+
+3. **Chunk store** (bulk, append-only). One record per release per batch:
+   its files' names, sizes and segment lists, compactly encoded (template
+   codec for message-ids, zstd). Written once, never updated. A batch is
+   stored in one transaction and only when it starts at its range's resume
+   point, so a batch sent twice or late is refused and the catalog's sums
+   stay right (no per-file dedup); a recovery run for a retracted rule only
+   adds what that rule had dropped. ~3 TB.
+
+4. **Release catalog** (hot, Postgres on NVMe). One row per release: key,
+   name, poster, groups, posted_at, file count, size, segments, expected
+   file count (`[i/N]`), has-par2, readable-name flag, state. Updated with
+   aggregates once per batch (one upsert per release per batch), plus a
+   small name index for search. ~115 M rows, ~50–100 GB.
+
+5. **Rules and recovery**. What a worker skips isn't stored (message-ids
+   are the bulk: garbage alone would be ~2.5 TB); a skipped range is read
+   again instead (headers are cheap: ~53 B per overview line compressed).
+   - *Per range, per rule and signature*: how many articles and files the
+     rule dropped (garbage per (tool, group), per-article random subjects,
+     obfuscated subjects, and crossposts per owner group). A few rows per
+     range.
+   - *Rule versions*: every range records the version of the rule set it ran
+     with. Retracting or changing a rule makes the ranges where it matched
+     stale; the coordinator leases them again for that rule only, and the
+     worker returns just what the rule had dropped.
+   - *Sample*: ~1% of what each rule drops (by a hash of the release key) is
+     kept with full file details and probe candidates, so a rule can be
+     checked, and its losses measured in phase 2, before it is retracted.
+
+6. **Probe queue**. The candidates, in the catalog's database: one small row
+   each (release, kind, message-id, chunk reference), priority by the
+   release's state (settled first) and kind (random names before readable
+   ones). Phase 2 takes them from here; phase 1 only fills it.
+
+7. **Fragments**. Files crossing a range edge: kept by range and article
+   number; merged when the neighbouring range is done, then written as a
+   chunk. Never staged per segment.
+
+8. **Settling**. A release is settled when the ranges covering its posting
+   span are done and no chunk arrived for a day: completeness is computed
+   then (expected vs found files and segments). Phase 2 picks its work from
+   settled releases.
+
+9. **NZB and API**. NZB built from a release's chunks; a minimal Newznab
+   search (names) and get. Web UI later.
+
+## Group list
+
+Editable at any time: an added group gets its ranges planned, a removed one
+no new leases. Each group gets a priority number when it is added (the first
+list by size, later additions after it); a crosspost belongs to the indexed
+group in its Xref with the lowest number. So adding a group never takes
+articles from groups already done, and removing one re-reads only the ranges
+of other groups whose counts show crossposts skipped for it (part 5). Start:
+the 1000 largest `alt.binaries.*` groups by LIST ACTIVE (~266 k ranges of
+2 M articles; boneless alone 80 k).
+
+## Corpus
+
+The grouping sweeps' dataset (2026-10-12; LOG.md 2026-10-12 (12)): the XOVER
+chunks of ranges marked `capture`, kept on the HDD in
+`corpus/<group>/<start>-<end>/` (edge-bench's layout, `range.json` once
+complete), up to 300 GB. The prefetcher stores them as it fetches and reads
+them from there afterwards, so a reset re-indexes them without the news
+server. `deploy/corpus_sweep.py` marks more ranges each sweep (`capture`,
+spread over groups and eras, the top-100 groups first), replays every stored
+range with the deployed build's edge-bench (`replay`, ~seconds a range) and
+reports suspects and the change since the previous build (`report`). A rule
+change is judged on the whole corpus before a reset.
+
+## Phase 2
+
+Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
+reads the probe queue's first articles (`lone`, `par2`, `archive`) and
+stores per release whether it's passworded and its inner name, which search,
+the web interface and Newznab use.
+
+Built: the par2 merge (2026-10-12; `releases:merge`, the `merger` service,
+LOG.md 2026-10-12 (13)). The prober stores per probed file start the MD5 of
+its first 16 KiB (`file_hashes`) and per par2 body its set's file list
+(`par2_files`, `par2_sets`), which holds the same hash per file whatever the
+file was renamed to. Every minute the merger merges the releases of one
+group, posted within 48 hours of a release holding the set's par2, whose
+files the set describes, into the one with the most files (chunks,
+candidates, inspections, counts; the par2's shared name when none was
+readable); `release_merges` keeps what went where. Obfuscated uploads whose
+files were each a release of their own (`lone` candidates) become one.
+
+**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
+voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
+homelab: `voyager1-tables.dump` with 2.76 M spots, upload signatures,
+signature probes, dropped releases and groups; `voyager1-release-passwords.csv`
+with 68,032 releases with a password):
+
+| Feature | Voyager 1 | In voyager 2 |
+|---|---|---|
+| Spotnet spots | D29 | the spotter and the spot inspector (below); new spots only, the export's not imported |
+| Posted NZBs read into releases (also NZBs inside archives) | D26, D28 | not yet |
+| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
+| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
+| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
+| 7z listings | D19 | no: `7z-tail` candidates are queued, not probed |
+| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
+| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
+
+Live indexing (the tail) runs since 2026-10-09.
+
+**Junk** (voyager 1's second filter, D42). After the header pass's rules
+(garbage signatures, executables, per-article and obfuscated subjects), the
+judge (`releases:judge`, service `judge`) judges releases hourly by what the
+prober, the merges and the spots found, in this order (the first rule
+wins): *dead* (every probed first article gone), *fake* (a film, episode
+or music name whose content is a program: archive entries all
+executables, or the file itself one by name or by its first bytes, sniffed
+`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
+*locked* (passworded, no password from a spot, its NZB or the title),
+*unusable* (random names, no inner name, no par2 or inner-name merge, no
+spot, a day after settling). Verdicts (`release_verdicts`, with the rule
+set's version) hide releases from search (`judged` shows them); a verdict
+whose rule no longer matches is taken back. Nothing is dropped yet.
+
+**Sidecars** (the merger's third pass). An upload's .nfo, .sfv or .nzb
+that came apart from it (another name, another counter: "[00/80] … .nzb"
+beside "[01/94] … .part01.rar") was a release of its own. The settler
+queues small settled releases (≤ 3 files, < 1 MB) in `fold_candidates`;
+one whose files are all sidecars joins the poster's bigger release in the
+same group posted within 10 minutes: the one sharing a word of 4+ letters
+with its subject, else the one under the same counter total, else none
+(`release_merges.via` 'sidecar').
+
+**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two
+processes, one NNTP connection each:
+
+- **The spotter** (`spots:read`, service `spotter`) reads free.pt's
+  overviews after the last article it saw (`spot_state`; the first time the
+  last `SPOT_BACKFILL`, 20,000, ~3 months), every `SPOT_EVERY_SECONDS`
+  (120). For each spot it reads the full header: the signature is checked
+  there as Spotweb does (Spotnet's keys 2–4; self-signed key 7 by hashcash),
+  and the X-XML gives the description, the website and the NZB's articles.
+  A password the title (`{{pw}}`) or the description names
+  (`passwordFromSpot`) is kept with the spot, with its source. A
+  delete/dispose message marks its target `removed` (a personal one only
+  from the same spotter within 5 days) and takes its links and passwords
+  back from the releases.
+- **The spot inspector** (`spots:inspect`, service `spot-inspector`) looks
+  inside each listed spot, newest first: it reads the compressed NZB (≤
+  `SPOT_MAX_NZB_SEGMENTS` articles in alt.binaries.ftd), lists its files
+  in `spot_files` (named as the header pass names them, `parseSubject`),
+  and matches each file to the release the header pass built for it: the
+  same release identity (`releaseIdentity` over the file's subject and
+  poster) in one of its groups, the release posted nearest the file within
+  48 h. Matched releases are linked in `release_spots`. Placeholder NZBs
+  (`dummy`), gone articles (`nzb-gone`) and unreadable ones are noted on the
+  spot. Unmatched spots are matched again every hour; after
+  `SPOT_STANDALONE_AFTER_HOURS` (24) a spot becomes a release of its own
+  (source `spot`), whose NZB is the spot's (kept gzipped in `spots.nzb`).
+- **Passwords are kept apart** in `release_passwords` (release, password,
+  source: `nzb` for the NZB's `<meta type="password">`, else `spot-title` or
+  `spot-text`, and the spot), not on the catalog row; the catalog only says
+  `passworded`.
+
+## Decisions
+
+a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by
+   month, on the HDD; files on disk only if writes still limit. **Accepted.**
+b. A separate database on the same server. **Accepted.**
+c. Group list: editable, owner by priority number (above). **Accepted.**
+d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
+e. Headers through the coordinator's relay, with prefetch (part 1).
+   **Accepted.**
+f. Random-named releases (no file with a readable name) keep no chunk in
+   the header pass: their file keys and probe candidates only, until
+   phase 2 names them (bodies) and fetches their segments again. The full
+   history is ~3 TB with them, ~1.2–1.4 TB without (725 M of the 1.25 B
+   kept files; LOG.md 2026-10-09). **Accepted** 2026-10-09, not built yet.

adonisrc.ts

@@ -1,4 +1,4 @@
-import { indexEntities } from '@adonisjs/core'
+import { indexEntities } from '@adonisjs/core/generators'
 import { defineConfig } from '@adonisjs/core/app'
 import { generateRegistry } from '@tuyau/core/hooks'
 

app/controllers/access_tokens_controller.ts

@@ -17,7 +17,7 @@ export default class AccessTokensController {
   }
 
   async destroy({ auth }: HttpContext) {
-    const user = auth.getUserOrFail()
+    const user = auth.use('api').getUserOrFail()
     if (user.currentAccessToken) {
       await User.accessTokens.delete(user, user.currentAccessToken.identifier)
     }

app/controllers/new_account_controller.ts

@@ -1,13 +1,36 @@
 import User from '#models/user'
+import db from '@adonisjs/lucid/services/db'
 import { signupValidator } from '#validators/user'
 import type { HttpContext } from '@adonisjs/core/http'
 import UserTransformer from '#transformers/user_transformer'
+import AccessService from '#services/access_service'
+
+class InviteGone extends Error {}
 
 export default class NewAccountController {
-  async store({ request, serialize }: HttpContext) {
-    const { fullName, email, password } = await request.validateUsing(signupValidator)
+  async store({ request, response, serialize }: HttpContext) {
+    const { fullName, email, password, invite } = await request.validateUsing(signupValidator)
+    if (AccessService.signup === 'closed') return response.forbidden({ error: 'Signup is closed' })
+    if (AccessService.signup === 'invite' && !invite) return response.forbidden({ error: 'An invite code is needed to sign up' })
 
-    const user = await User.create({ fullName, email, password })
+    // The user and their invite together: an invite that's gone rolls the signup back.
+    const user = await db
+      .transaction(async (trx) => {
+        const created = await User.create({ fullName, email, password }, { client: trx })
+        if (AccessService.signup !== 'invite') return created
+        const gives = await AccessService.redeem(invite!, created.id, trx)
+        if (!gives) throw new InviteGone()
+        if (gives.tier) {
+          AccessService.grantTier(created, gives.tier, gives.tier_days)
+          await created.useTransaction(trx).save()
+        }
+        return created
+      })
+      .catch((error) => {
+        if (error instanceof InviteGone) return null
+        throw error
+      })
+    if (!user) return response.forbidden({ error: 'Unknown or used invite code' })
     const token = await User.accessTokens.create(user)
 
     return serialize({

app/controllers/newznab_controller.ts

@@ -1,6 +1,8 @@
 import type { HttpContext } from '@adonisjs/core/http'
 import env from '#start/env'
-import SearchService from '#services/SearchService'
+import NzbService from '#services/nzb_service'
+import SearchService from '#services/search_service'
+import AccessService, { AccessDenied } from '#services/access_service'
 import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
 
 const LIMITS = { max: 100, default: 50 }
@@ -17,7 +19,7 @@ const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
   'book': 7000,
 }
 
-/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */
+/** The configured key, if any. */
 function configuredApiKey(): string | undefined {
   const value: any = env.get('NEWZNAB_API_KEY')
   const key = typeof value === 'string' ? value : value?.release()
@@ -29,29 +31,43 @@ function int(value: unknown): number | undefined {
 }
 
 /**
- * Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like:
+ * Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1):
  * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
+ * Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
  */
 export default class NewznabController {
-  async handle({ request, response }: HttpContext) {
+  async handle(ctx: HttpContext) {
+    const xml = (body: string, type = 'application/xml') => {
+      ctx.response.header('Content-Type', `${type}; charset=utf-8`)
+      return ctx.response.send(body)
+    }
+    try {
+      return await this.answer(ctx, xml)
+    } catch (error) {
+      if (error instanceof AccessDenied) return xml(errorXml(error.newznabCode, error.message))
+      throw error
+    }
+  }
+
+  private async answer(ctx: HttpContext, xml: (body: string, type?: string) => void) {
+    const { request, response } = ctx
     const qs = request.qs()
     const t = String(qs.t ?? '')
-    const server = this.server(request)
 
-    const xml = (body: string, type = 'application/xml') => {
-      response.header('Content-Type', `${type}; charset=utf-8`)
-      return response.send(body)
-    }
-
-    const apiKey = configuredApiKey()
-    if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
-
-    if (t === 'caps') return xml(capsXml(server, LIMITS))
+    // With accounts the key is a user's (AccessService); else the one configured, if any.
+    const grant = await AccessService.authorize(ctx, qs.apikey)
+    const apiKey = grant ? grant.user.apiKey : configuredApiKey()
+    if (!grant && apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
+    const server = this.server(request, apiKey)
+    if (t === 'caps') return xml(capsXml(server, LIMITS, { available: AccessService.accounts && AccessService.signup !== 'closed', open: AccessService.accounts && AccessService.signup === 'open' }))
 
     if (t === 'get') {
       const id = int(qs.id)
       if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
-      const nzb = await SearchService.nzb(id)
+      // Unknown ids and releases past the tier's retention aren't counted.
+      if (!(await AccessService.checkRelease(grant, id))) return xml(errorXml(300, 'No such item'))
+      await AccessService.charge(grant, 'grab')
+      const nzb = await NzbService.build(id)
       if (!nzb) return xml(errorXml(300, 'No such item'))
       const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
       response.header('Content-Disposition', `attachment; filename="${filename}"`)
@@ -59,6 +75,7 @@ export default class NewznabController {
     }
 
     if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
+    await AccessService.charge(grant, 'api')
 
     const requested = String(qs.cat ?? '')
       .split(',')
@@ -69,20 +86,17 @@ export default class NewznabController {
     const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
     const offset = int(qs.offset) ?? 0
 
-    // ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but
-    // answer with an empty result rather than everything.
+    // ID-only lookups (tvdbid, imdbid, …) aren't supported: an empty result rather than everything.
     const q = typeof qs.q === 'string' ? qs.q.trim() : ''
     const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
-
     const result = idOnly
-      ? { total: 0, releases: [] }
+      ? null
       : await SearchService.search({
           q,
           categories,
           readable: true,
           notIncomplete: true,
-          hideContainers: true,
-          maxAgeDays: int(qs.maxage),
+          maxAgeDays: AccessService.maxAgeDays(grant, int(qs.maxage)),
           season: int(qs.season),
           episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
           sort: 'newest',
@@ -90,32 +104,38 @@ export default class NewznabController {
           offset,
         })
 
-    const items = result.releases.map((r: any) => {
+    const items = (result?.releases ?? []).map((r) => {
       const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
       return {
         id: r.id,
-        title: r.name,
+        // An obfuscated release by what the prober found inside, without its extension (what clients parse).
+        title: r.readable || !r.innerName ? r.name : r.innerName.replace(/\.[a-z0-9]{2,4}$/i, ''),
         category: r.category ?? 8010,
         size: r.size,
         files: r.fileCount,
         postedAt: r.postedAt ? new Date(r.postedAt) : null,
         poster: r.poster,
         groups: r.groups ?? [],
-        password: Boolean(r.encrypted),
+        // From the prober (phase 2): encrypted archive entries or headers.
+        password: r.passworded === true,
         season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
         episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
       }
     })
-    return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml')
+    // What's left of the day, for clients that show it (newznab:apilimits).
+    const limits = grant && {
+      apiCurrent: grant.usage.apiHits,
+      apiMax: grant.limits.apiHits,
+      grabCurrent: grant.usage.grabs,
+      grabMax: grant.limits.grabs,
+    }
+    return xml(resultsXml(server, items, offset, result?.total ?? 0, limits || undefined), 'application/rss+xml')
   }
 
-  private server(request: HttpContext['request']): NewznabServer {
+  /** Download links carry the caller's key. */
+  private server(request: HttpContext['request'], apiKey: string | undefined): NewznabServer {
     const proto = request.header('x-forwarded-proto') ?? 'http'
     const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
-    return {
-      title: 'Usenet Indexer',
-      baseUrl: `${proto}://${host}`,
-      apiKey: configuredApiKey(),
-    }
+    return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey }
   }
 }

app/controllers/node_controller.ts

@@ -0,0 +1,112 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import { Readable } from 'node:stream'
+import { setTimeout as sleep } from 'node:timers/promises'
+import Leases, { LeaseLost } from '#services/leases'
+import BatchStore, { StaleBatch } from '#services/batch_store'
+import Spool from '#services/spool'
+import { KEEPALIVE, encodeFrame, fromWire, type WireBatch } from '#lib/edge/index'
+
+/**
+ * How long the stream waits for a chunk the prefetcher hasn't spooled yet. Long: a chunk that keeps
+ * coming in isn't given up (only a stalled one is), and some groups' chunks take 20+ minutes
+ * (a.b.0day.stuffz, 17 MB in 22 min; LOG.md 2026-10-10 (3)).
+ */
+const CHUNK_WAIT_MS = 60 * 60 * 1000
+/** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
+const KEEPALIVE_MS = 20_000
+
+/**
+ * The edge workers' API (REBUILD.md, parts 1–2): a lease, the range's
+ * chunks as one stream, results per batch, and the end of the range.
+ */
+export default class NodeController {
+  private node(ctx: HttpContext) {
+    return Number(ctx.auth.use('node').getUserOrFail().id)
+  }
+
+  async lease(ctx: HttpContext) {
+    const lease = await Leases.lease(this.node(ctx))
+    if (!lease) return ctx.response.noContent()
+    return lease
+  }
+
+  /** The range's chunks from its resume point, waiting for those still being fetched. */
+  async stream(ctx: HttpContext) {
+    const nodeId = this.node(ctx)
+    const rangeId = Number(ctx.params.id)
+    const range = await this.leased(ctx, rangeId, nodeId)
+    if (!range) return
+    const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
+    async function* frames() {
+      for (const span of spans) {
+        const waitUntil = Date.now() + CHUNK_WAIT_MS
+        let quietSince = Date.now()
+        while (!(await Spool.has(rangeId, span))) {
+          if (Date.now() > waitUntil) throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} wasn't spooled in time`)
+          if (Date.now() - quietSince >= KEEPALIVE_MS) {
+            quietSince = Date.now()
+            yield encodeFrame(KEEPALIVE)
+            await Leases.heartbeat(rangeId, nodeId)
+          }
+          await sleep(500)
+        }
+        yield encodeFrame({ ...span, data: await Spool.read(rangeId, span) })
+        // Streaming keeps the lease alive.
+        await Leases.heartbeat(rangeId, nodeId)
+      }
+    }
+    ctx.response.header('Content-Type', 'application/octet-stream')
+    ctx.response.stream(Readable.from(frames()))
+  }
+
+  async batch(ctx: HttpContext) {
+    const nodeId = this.node(ctx)
+    const rangeId = Number(ctx.params.id)
+    try {
+      return await BatchStore.store(rangeId, nodeId, fromWire(ctx.request.body() as WireBatch))
+    } catch (error) {
+      if (error instanceof StaleBatch) return ctx.response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  async heartbeat(ctx: HttpContext) {
+    return this.lifecycle(ctx, (rangeId, nodeId) => Leases.heartbeat(rangeId, nodeId))
+  }
+
+  async finish(ctx: HttpContext) {
+    return this.lifecycle(ctx, (rangeId, nodeId) => Leases.finish(rangeId, nodeId))
+  }
+
+  /** POST /ranges/:id/corrupt { from }: a chunk that doesn't inflate; the range goes back. */
+  async corrupt(ctx: HttpContext) {
+    const from = Number(ctx.request.input('from'))
+    return this.lifecycle(ctx, (rangeId, nodeId) => Leases.corrupt(rangeId, nodeId, from))
+  }
+
+  async release(ctx: HttpContext) {
+    return this.lifecycle(ctx, (rangeId, nodeId) => Leases.release(rangeId, nodeId))
+  }
+
+  private async lifecycle(ctx: HttpContext, action: (rangeId: number, nodeId: number) => Promise<void>) {
+    try {
+      await action(Number(ctx.params.id), this.node(ctx))
+      return { ok: true }
+    } catch (error) {
+      if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
+      throw error
+    }
+  }
+
+  private async leased(ctx: HttpContext, rangeId: number, nodeId: number) {
+    try {
+      return await Leases.leased(rangeId, nodeId)
+    } catch (error) {
+      if (error instanceof LeaseLost) {
+        ctx.response.conflict({ error: error.message })
+        return null
+      }
+      throw error
+    }
+  }
+}

app/controllers/profile_controller.ts

@@ -1,8 +1,19 @@
 import UserTransformer from '#transformers/user_transformer'
 import type { HttpContext } from '@adonisjs/core/http'
+import AccessService from '#services/access_service'
 
 export default class ProfileController {
+  /** The user, their tier, Newznab key and what they used today. */
   async show({ auth, serialize }: HttpContext) {
-    return serialize(UserTransformer.transform(auth.getUserOrFail()))
+    const user = auth.use('api').getUserOrFail()
+    return serialize({ user: UserTransformer.transform(user), access: await AccessService.summary(user) })
+  }
+
+  /** POST /api/v1/account/api-key: a new Newznab key (the old one stops working). */
+  async resetApiKey({ auth }: HttpContext) {
+    const user = auth.use('api').getUserOrFail()
+    user.apiKey = AccessService.newKey()
+    await user.save()
+    return { apiKey: user.apiKey }
   }
 }

app/controllers/releases_controller.ts

@@ -1,35 +1,63 @@
 import type { HttpContext } from '@adonisjs/core/http'
-import SearchService, { type SortOrder } from '#services/SearchService'
+import NzbService from '#services/nzb_service'
+import AccessService from '#services/access_service'
+import SearchService, { type SortOrder } from '#services/search_service'
+import { expandCategories } from '#lib/newznab/index'
 
 const flag = (value: unknown) => value === '1' || value === 'true' || value === true
 
+/** A size in MB from the query string, as bytes. */
+function megabytes(value: unknown): number | undefined {
+  const mb = Number(value)
+  return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
+}
+
+/** The catalog for the web interface (REBUILD.md, part 9). */
 export default class ReleasesController {
-  /** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&source=&poster=&sort=&page=&per_page= */
-  async index({ request }: HttpContext) {
-    const qs = request.qs()
+  /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
+  async index(ctx: HttpContext) {
+    const qs = ctx.request.qs()
+    const grant = await AccessService.authorize(ctx, qs.apikey)
+    await AccessService.charge(grant, 'api')
+    const categories = String(qs.cat ?? '')
+      .split(',')
+      .map((c) => Number(c.trim()))
+      .filter((c) => Number.isInteger(c) && c > 0)
     return SearchService.search({
+      maxAgeDays: AccessService.maxAgeDays(grant),
       q: typeof qs.q === 'string' ? qs.q : undefined,
+      categories: categories.length ? expandCategories(categories) : undefined,
       readable: flag(qs.readable),
       complete: flag(qs.complete),
-      hideEncrypted: flag(qs.hide_encrypted),
-      includePar2Only: flag(qs.par2_only),
+      unlocked: flag(qs.unlocked),
       source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
       poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
+      minSize: megabytes(qs.min_size),
+      maxSize: megabytes(qs.max_size),
       sort: qs.sort as SortOrder | undefined,
       page: Number(qs.page) || 1,
       perPage: Number(qs.per_page) || 50,
+      // The page shows "1,000+"; counting further made filter clicks slow.
+      countLimit: 1000,
     })
   }
 
-  /** GET /api/v1/releases/:id */
-  async show({ params, response }: HttpContext) {
-    const release = await SearchService.release(Number(params.id))
-    return release ?? response.notFound({ error: 'Release not found' })
+  /** GET /api/v1/releases/:id, with its files */
+  async show(ctx: HttpContext) {
+    const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
+    if (!(await AccessService.checkRelease(grant, Number(ctx.params.id)))) return ctx.response.notFound({ error: 'Release not found' })
+    await AccessService.charge(grant, 'api')
+    const release = await SearchService.release(Number(ctx.params.id))
+    return release ?? ctx.response.notFound({ error: 'Release not found' })
   }
 
-  /** GET /api/v1/releases/:id/nzb */
-  async nzb({ params, response }: HttpContext) {
-    const nzb = await SearchService.nzb(Number(params.id))
+  /** GET /api/v1/releases/:id/nzb (?apikey= for a plain link, as the web interface's) */
+  async nzb(ctx: HttpContext) {
+    const { params, response } = ctx
+    const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
+    if (!(await AccessService.checkRelease(grant, Number(params.id)))) return response.notFound({ error: 'Release not found' })
+    await AccessService.charge(grant, 'grab')
+    const nzb = await NzbService.build(Number(params.id))
     if (!nzb) return response.notFound({ error: 'Release not found' })
     const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
     response.header('Content-Type', 'application/x-nzb')
@@ -37,7 +65,12 @@ export default class ReleasesController {
     return response.send(nzb.xml)
   }
 
-  /** GET /api/v1/stats */
+  /** GET /api/v1/access: whether the web interface needs an account, and how one is made. */
+  async access() {
+    return { mode: AccessService.mode, signup: AccessService.signup }
+  }
+
+  /** GET /api/v1/stats (open: what the index holds, not what's in it) */
   async stats() {
     return SearchService.stats()
   }

app/exceptions/handler.ts

@@ -8,6 +8,9 @@ export default class HttpExceptionHandler extends ExceptionHandler {
    */
   protected debug = !app.inProduction
 
+  /** A refused request (AccessService: no key, a limit reached) is an answer, not a fault. */
+  protected ignoreCodes = ['E_ACCESS_DENIED']
+
   /**
    * The method is used for handling errors and returning
    * response to the client

app/models/base_model.ts

@@ -1,4 +0,0 @@
-import { BaseModel as AdonisBaseModel } from '@adonisjs/lucid/orm'
-
-export default class BaseModel extends AdonisBaseModel {
-}

app/models/file.ts

@@ -1,35 +0,0 @@
-import { DateTime } from 'luxon'
-import { column } from '@adonisjs/lucid/orm'
-import BaseModel from '#models/base_model'
-
-export default class File extends BaseModel {
-  @column({ isPrimary: true })
-  declare id: number
-
-  @column({ columnName: 'file_key' })
-  declare fileKey: string | null
-
-  @column()
-  declare filename: string
-
-  @column()
-  declare poster: string
-
-  @column()
-  declare date: number
-
-  @column()
-  declare parts: number
-
-  @column({ columnName: 'message_ids' })
-  declare messageIds: Record<string, any>
-
-  @column()
-  declare groups: string[]
-
-  @column.dateTime({ autoCreate: true })
-  declare createdAt: DateTime
-
-  @column.dateTime({ autoCreate: true, autoUpdate: true })
-  declare updatedAt: DateTime
-}

app/models/group.ts

@@ -1,32 +0,0 @@
-import { DateTime } from 'luxon'
-import { column } from '@adonisjs/lucid/orm'
-import BaseModel from '#models/base_model'
-
-export default class Group extends BaseModel {
-  @column({ isPrimary: true })
-  declare id: number
-
-  @column()
-  declare name: string
-
-  @column()
-  declare active: boolean
-
-  /** Every article up to here was fetched successfully (see FetchRangeService). */
-  @column({ columnName: 'last_indexed_id' })
-  declare lastIndexedId: bigint
-
-  /** Download bodies of obfuscated articles to identify them (bandwidth-heavy). */
-  @column({ columnName: 'fetch_bodies' })
-  declare fetchBodies: boolean
-
-  /** How far the scheduler has queued fetch jobs. */
-  @column({ columnName: 'last_queued_id' })
-  declare lastQueuedId: bigint
-
-  @column.dateTime({ autoCreate: true })
-  declare createdAt: DateTime
-
-  @column.dateTime({ autoCreate: true, autoUpdate: true })
-  declare updatedAt: DateTime
-}

app/models/node.ts

@@ -0,0 +1,8 @@
+import { NodeSchema } from '#database/schema'
+import { type AccessToken, DbAccessTokensProvider } from '@adonisjs/auth/access_tokens'
+
+/** An edge worker (REBUILD.md, part 2); it authenticates with an access token (the `node` guard). */
+export default class Node extends NodeSchema {
+  static accessTokens = DbAccessTokensProvider.forModel(Node, { table: 'node_access_tokens' })
+  declare currentAccessToken?: AccessToken
+}

app/models/user.ts

@@ -1,4 +1,6 @@
+import { randomBytes } from 'node:crypto'
 import { UserSchema } from '#database/schema'
+import { beforeCreate } from '@adonisjs/lucid/orm'
 import hash from '@adonisjs/core/services/hash'
 import { compose } from '@adonisjs/core/helpers'
 import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
@@ -8,6 +10,12 @@ export default class User extends compose(UserSchema, withAuthFinder(hash)) {
   static accessTokens = DbAccessTokensProvider.forModel(User)
   declare currentAccessToken?: AccessToken
 
+  /** The Newznab key (AccessService), known on the model right away rather than from the column's default. */
+  @beforeCreate()
+  static assignApiKey(user: User) {
+    user.apiKey ??= randomBytes(16).toString('hex')
+  }
+
   get initials() {
     const [first, last] = this.fullName ? this.fullName.split(' ') : this.email.split('@')
     if (first && last) {

app/services/ArchiveService.ts

@@ -1,160 +0,0 @@
-import { execFile } from 'node:child_process'
-import { mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import path from 'node:path'
-import { promisify } from 'node:util'
-import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import IngestService from '#services/IngestService'
-import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem, rarReleaseIdentity } from '#lib/release/index'
-
-/** Stored per file and release; a listing of thousands of files is not useful. */
-const MAX_ENTRIES = 500
-
-export interface ArchiveResult {
-  info: RarInfo
-  releaseId?: number
-  releaseRenamed?: string
-  grouped?: boolean
-}
-
-/**
- * Records what a RAR volume's headers say: the listing on the file, the
- * release it belongs to (from the shared directory or continued entry, when
- * the file has none yet), and for first volumes the contents and password
- * flag on the release. See CONCEPT.md (D19, D21).
- */
-class ArchiveService {
-  async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
-    const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
-    await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) })
-
-    let releaseId = file.release_id ?? binary?.release_id ?? null
-    let grouped = false
-    const identity = rarReleaseIdentity(info.entries)
-    if (!releaseId && identity) {
-      const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
-        {
-          release: identity,
-          poster: binary?.poster ?? file.poster,
-          weakIdentity: true,
-          groups: binary?.groups ?? file.groups ?? [],
-          postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
-        },
-      ])
-      releaseId = ids.get(identity.key)!
-      grouped = true
-      if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
-      await db.from('files').where('id', file.id).update({ release_id: releaseId })
-    }
-    if (!releaseId) return { info }
-
-    const release = await db.from('releases').where('id', releaseId).first()
-    const update: Record<string, unknown> = {}
-    if (info.firstVolume !== false) {
-      update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
-      if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
-    } else if (info.headersEncrypted) {
-      update.encrypted = true
-    }
-    const better = releaseNameFromContents(info)
-    const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
-    if (renamed) update.name = renamed
-    if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
-
-    if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
-    return { info, releaseId, releaseRenamed: renamed, grouped }
-  }
-}
-
-const run = promisify(execFile)
-
-/** Tools that print one archive entry to stdout, tried in order. */
-const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
-  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
-  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
-]
-
-/**
- * The unpacked data of one entry of a complete (single-volume) archive:
- * copied out when stored, else unpacked with bsdtar (libarchive, in the
- * Docker image) or unrar. Null when that is not possible.
- */
-export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
-  const stored = storedEntryData(archive, entry)
-  if (stored) return stored
-  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
-  if (entry.size !== null && entry.size > maxBytes) return null
-
-  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
-  try {
-    const file = path.join(dir, 'archive.rar')
-    await writeFile(file, archive)
-    for (const [tool, args] of EXTRACTORS) {
-      try {
-        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
-        if (stdout.length > 0) return stdout
-      } catch (error: any) {
-        if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
-      }
-    }
-    return null
-  } finally {
-    await rm(dir, { recursive: true, force: true })
-  }
-}
-
-/**
- * A release name from an archive listing: the top-level directory shared by
- * all entries when it reads like a release name, otherwise the stem of the
- * largest entry with a readable name (e.g. an archive holding just
- * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
- */
-export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
-  const entries = info.entries.filter((e) => !e.directory)
-  const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0]))
-  if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) {
-    const [top] = tops
-    if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
-  }
-  const candidates = entries
-    .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
-    .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
-    .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
-    .sort((a, b) => b.size - a.size)
-  return candidates[0]?.stem
-}
-
-/**
- * Renames random-named releases from their stored archive listing (no
- * downloads). Used by `release:rename`.
- */
-export async function renameFromStoredContents(): Promise<number> {
-  const releases = await db
-    .from('releases')
-    .whereNotNull('contents')
-    .whereRaw("jsonb_array_length(contents) > 0")
-    .select('id', 'name', 'source', 'contents')
-  let renamed = 0
-  for (const release of releases) {
-    if (!looksRandom(release.name) && release.source !== 'post') continue
-    const better = releaseNameFromContents({ entries: release.contents })
-    if (!better || better === release.name) continue
-    await db.from('releases').where('id', release.id).update({ name: better })
-    renamed++
-  }
-  return renamed
-}
-
-/** Obfuscated names: one long token of mixed letters and digits, no separators. */
-export function looksRandom(name: string): boolean {
-  // An unknown short extension (e.g. ".bin") says nothing about the name.
-  name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
-  if (/^[0-9a-f]{16,}$/i.test(name)) return true
-  if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
-  // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
-  return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
-}
-
-export default new ArchiveService()

app/services/ContentService.ts

@@ -1,187 +0,0 @@
-import { createHash } from 'node:crypto'
-import db from '@adonisjs/lucid/services/db'
-import NntpService from '#services/NntpService'
-import Par2Service, { type Par2Result } from '#services/Par2Service'
-import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
-import NameService from '#services/NameService'
-import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
-import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
-import { sniffContentType, type ContentType } from '#lib/content/index'
-import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem } from '#lib/release/index'
-import { looksLikeFilename } from '#lib/subject/index'
-
-const FIRST_CHUNK = 16 * 1024
-
-/**
- * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
- * the file's first part. Undefined when that part is shorter than 16 KiB but
- * the file is not (the chunk would span two parts).
- */
-export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
-  return chunkMd5(decodeSegment(body).data, fileSize)
-}
-
-function chunkMd5(data: Buffer, fileSize: number): string | undefined {
-  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
-  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
-}
-
-export interface ContentResult {
-  par2?: Par2Result
-  archive?: ArchiveResult
-  named: number
-  contentType: ContentType | null
-  nzb?: NzbImportResult
-  /** NZBs read from inside an archive. */
-  archiveNzbs?: NzbImportResult[]
-}
-
-/**
- * Takes names and groupings from file contents (CONCEPT.md D21). For a
- * completed file with a random name, one article (the first) is enough:
- * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
- * - par2 files are recognised by content and name every file of their set,
- * - RAR headers give the release grouping, name and contents.
- */
-class ContentService {
-  /** Random or missing names, and named first RAR volumes (for contents and password flag). */
-  isCandidate(name: string) {
-    if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
-    if (/\.nzb$/i.test(name)) return true
-    return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
-  }
-
-  /** Downloads a whole (small) file whose first segment is already decoded. */
-  private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
-    const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
-    const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
-    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
-    if (parts.length === 1) return first
-
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const [, part] of parts) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    return assembled!.getBuffer()
-  }
-
-  /**
-   * Downloads a small archive that lists NZBs and imports each of them
-   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
-   * uploads by other posters.
-   */
-  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
-    const archive = await this.readWhole(first, file.message_ids)
-    const info = archive && parseRarHeaders(archive)
-    if (!archive || !info) return []
-    const entries = info.entries.filter(isNzbEntry)
-    const results: NzbImportResult[] = []
-    for (const entry of entries) {
-      const xml = await extractEntry(archive, entry)
-      if (!xml) continue
-      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
-        entryName: entry.name,
-        moveContainer: entries.length === 1,
-      })
-      if (imported) results.push(imported)
-    }
-    return results
-  }
-
-  async inspect(fileId: number): Promise<ContentResult | null> {
-    const file = await db.from('files').where('id', fileId).first()
-    const binary = await db.from('binaries').where('file_id', fileId).first()
-    if (!file || !this.isCandidate(file.filename)) return null
-    const first = file.message_ids?.['1']
-    if (!first?.id) return null
-
-    const conn = await NntpService.acquire()
-    let failed = false
-    let body: Buffer
-    try {
-      body = await NntpService.fetchBody(conn, first.id)
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
-    let segment
-    try {
-      segment = decodeSegment(body)
-    } catch (error) {
-      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
-      segment = decodeSegment(body, { lenientSize: true })
-    }
-    const size = segment.meta.begin.size
-    const contentType = sniffContentType(segment.data)
-    await db.from('files').where('id', fileId).update({ content_type: contentType })
-
-    // Fingerprint, so par2 sets (now or later) can name this file exactly.
-    if (binary) {
-      await db
-        .from('binaries')
-        .where('id', binary.id)
-        .update({
-          size: db.raw('coalesce(size, ?)', [size]),
-          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
-        })
-    }
-
-    const result: ContentResult = { named: 0, contentType }
-    if (contentType === 'nzbe' || contentType === 'rclone') {
-      // An encrypted NZB, or rclone crypt (names and data encrypted): nothing
-      // to read, and useless without the key.
-      const releaseId = file.release_id ?? binary?.release_id
-      if (releaseId) {
-        await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
-      }
-      return result
-    }
-    if (contentType === 'nzb') {
-      const xml = await this.readWhole(segment.data, file.message_ids)
-      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
-      return result
-    }
-    const set = await Par2Service.read(segment.data, file.message_ids)
-    if (set && binary) {
-      result.par2 = await Par2Service.attach(binary, set)
-    } else {
-      const info = parseRarHeaders(segment.data)
-      if (info) {
-        result.archive = await ArchiveService.record(file, binary, info)
-        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
-      }
-    }
-    if (binary) result.named = await NameService.resolve([binary.id])
-    return result
-  }
-}
-
-/** NZBs that can be read from a single, unencrypted archive. */
-function hasNzbEntries(info: RarInfo): boolean {
-  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
-}
-
-function isNzbEntry(entry: RarEntry): boolean {
-  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
-}
-
-/** Small posted files (NZBs) that are read completely. */
-const WHOLE_MAX_PARTS = 40
-const WHOLE_MAX_BYTES = 25 * 1024 * 1024
-
-export default new ContentService()

app/services/DeferredService.ts

@@ -1,264 +0,0 @@
-import { createHash } from 'node:crypto'
-import db from '@adonisjs/lucid/services/db'
-import indexerConfig from '#config/indexer'
-import NntpService from '#services/NntpService'
-import QueueService from '#services/QueueService'
-import { type OverviewHeader } from '#services/IngestService'
-import { decodeSegment, inferTotalParts, parseYencMeta } from '#lib/yenc/index'
-import { sniffContentType } from '#lib/content/index'
-import { parseRarHeaders } from '#lib/rar/index'
-
-/** Expanding re-reads the file's article range with XOVER; beyond this it is too costly. */
-const EXPAND_MAX_RANGE = 200_000
-/** A claimed row whose worker died is taken again after this long. */
-const RECLAIM_AFTER = '1 hour'
-
-export interface DeferredArticle {
-  articleNumber: number
-  header: OverviewHeader
-}
-
-/** File name from a deferred uploader's subject: `[i/M] "name" yEnc`, no part counter. */
-export function deferredName(subject: string | undefined): string | null {
-  return /"([^"]+)"/.exec(subject ?? '')?.[1]?.trim() || null
-}
-
-export function isDeferred(header: OverviewHeader): boolean {
-  const id = header['message-id'] ?? ''
-  return indexerConfig.deferBodyMessageIds.some((re) => re.test(id))
-}
-
-/** A recorded range never spans more articles than this (one XOVER to expand it). */
-const RANGE_MAX_SPAN = 50_000
-
-export function bodyJob(header: OverviewHeader, group: string, articleNumber: number | bigint) {
-  return {
-    name: 'process-body',
-    data: { header, group, articleNumber: Number(articleNumber) },
-    // By message-id: an article crossposted to several indexed groups is downloaded once.
-    opts: { jobId: `body-${createHash('sha1').update(header['message-id'] ?? '').digest('hex')}` },
-  }
-}
-
-/**
- * Deferred uploads (CONCEPT.md D28). Their articles skip the body queue and
- * are recorded here per file, from the headers alone. `worker:deferred`
- * later downloads one body per file for its size, part count and content
- * type, and sends small files (NZBs in RARs, par2 index files) to the body
- * queue completely.
- */
-class DeferredService {
-  /**
-   * Records one group's deferred articles: per file when the subject names
-   * the file, otherwise only as article ranges. Returns the number of files.
-   */
-  async record(groupName: string, articles: DeferredArticle[]): Promise<number> {
-    const named = articles.filter((a) => deferredName(a.header.subject) !== null)
-    await this.recordRanges(groupName, articles.filter((a) => deferredName(a.header.subject) === null))
-    return this.recordFiles(groupName, named)
-  }
-
-  private async recordRanges(groupName: string, articles: DeferredArticle[]) {
-    if (articles.length === 0) return
-    const sorted = [...articles].sort((a, b) => Number(a.articleNumber) - Number(b.articleNumber))
-    const now = new Date()
-    const rows: Record<string, unknown>[] = []
-    for (const article of sorted) {
-      const n = Number(article.articleNumber)
-      const bytes = Number.parseInt(article.header[':bytes'] ?? '0', 10) || 0
-      const last = rows.at(-1)
-      if (last && n - (last.first_article as number) <= RANGE_MAX_SPAN) {
-        last.last_article = n
-        ;(last.article_count as number)++
-        last.bytes = (last.bytes as number) + bytes
-      } else {
-        rows.push({ group_name: groupName, first_article: n, last_article: n, article_count: 1, bytes, created_at: now })
-      }
-    }
-    await db.table('deferred_ranges').multiInsert(rows)
-  }
-
-  private async recordFiles(groupName: string, articles: DeferredArticle[]): Promise<number> {
-    const files = new Map<string, {
-      name: string; subject: string; poster: string; first: number; firstId: string; last: number
-      count: number; bytes: number; postedAt: Date | null
-    }>()
-    for (const { articleNumber, header } of articles) {
-      const name = deferredName(header.subject)!
-      const n = Number(articleNumber)
-      const bytes = Number.parseInt(header[':bytes'] ?? '0', 10) || 0
-      const date = header.date ? new Date(header.date) : null
-      const postedAt = date && !Number.isNaN(date.getTime()) ? date : null
-      const file = files.get(name)
-      if (!file) {
-        files.set(name, {
-          name, subject: header.subject ?? '', poster: header.from ?? '', first: n,
-          firstId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), last: n, count: 1, bytes, postedAt,
-        })
-        continue
-      }
-      if (n < file.first) {
-        file.first = n
-        file.firstId = (header['message-id'] ?? '').replace(/^<|>$/g, '')
-      }
-      file.last = Math.max(file.last, n)
-      file.count++
-      file.bytes += bytes
-      if (postedAt && (!file.postedAt || postedAt < file.postedAt)) file.postedAt = postedAt
-    }
-    if (files.size === 0) return 0
-
-    const now = new Date()
-    const rows = [...files.values()].map((f) => ({
-      file_key: createHash('sha1').update(`deferred\n${f.name}`).digest('hex'),
-      name: f.name,
-      subject: f.subject,
-      poster: f.poster,
-      group_name: groupName,
-      first_article: f.first,
-      first_message_id: f.firstId,
-      last_article: f.last,
-      article_count: f.count,
-      bytes: f.bytes,
-      posted_at: f.postedAt,
-      first_seen_at: now,
-      last_seen_at: now,
-    }))
-    // Counts and the article range only add up within the group the file was first seen in.
-    await db.rawQuery(
-      `INSERT INTO deferred_files AS d (${Object.keys(rows[0]).join(', ')})
-       SELECT * FROM jsonb_to_recordset(?::jsonb) AS r(
-         file_key varchar, name text, subject text, poster text, group_name text, first_article bigint,
-         first_message_id text, last_article bigint, article_count int, bytes bigint, posted_at timestamptz,
-         first_seen_at timestamptz, last_seen_at timestamptz)
-       ON CONFLICT (file_key) DO UPDATE SET
-         last_seen_at = EXCLUDED.last_seen_at,
-         first_message_id = CASE WHEN d.group_name = EXCLUDED.group_name AND EXCLUDED.first_article < d.first_article
-           THEN EXCLUDED.first_message_id ELSE d.first_message_id END,
-         first_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN LEAST(d.first_article, EXCLUDED.first_article) ELSE d.first_article END,
-         last_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN GREATEST(d.last_article, EXCLUDED.last_article) ELSE d.last_article END,
-         article_count = d.article_count + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.article_count ELSE 0 END,
-         bytes = d.bytes + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.bytes ELSE 0 END,
-         posted_at = LEAST(d.posted_at, EXCLUDED.posted_at)`,
-      [JSON.stringify(rows)]
-    )
-    return files.size
-  }
-
-  /** Takes up to `limit` files that stopped growing; they are marked `processing`. */
-  async claim(limit: number): Promise<any[]> {
-    const settled = new Date(Date.now() - indexerConfig.deferredSettleMs)
-    const { rows } = await db.rawQuery(
-      `UPDATE deferred_files SET status = 'processing', processed_at = now()
-       WHERE id IN (
-         SELECT id FROM deferred_files
-         WHERE (status = 'pending' AND last_seen_at < ?)
-            OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
-         ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED)
-       RETURNING *`,
-      [settled, limit]
-    )
-    return rows
-  }
-
-  /**
-   * Queues the bodies of up to `limit` recorded ranges, oldest first: the
-   * articles are found again with XOVER. Returns ranges and articles queued.
-   */
-  async expandRanges(limit: number): Promise<{ ranges: number; articles: number }> {
-    const ranges = await db.from('deferred_ranges').whereNull('expanded_at').orderBy('id').limit(limit)
-    let articles = 0
-    for (const range of ranges) {
-      const conn = await NntpService.acquire()
-      let failed = false
-      let overviews: [number, OverviewHeader][]
-      try {
-        await conn.group(range.group_name)
-        overviews = (await conn.xover(String(range.first_article), String(range.last_article))).overviews
-      } catch (error) {
-        failed = true
-        throw error
-      } finally {
-        NntpService.release(conn, failed)
-      }
-      const jobs = overviews
-        .filter(([, header]) => header && isDeferred(header) && deferredName(header.subject) === null)
-        .map(([articleNumber, header]) => bodyJob(header, range.group_name, articleNumber))
-      for (let i = 0; i < jobs.length; i += 5000) await QueueService.bodyQueue.addBulk(jobs.slice(i, i + 5000))
-      await db.from('deferred_ranges').where('id', range.id).update({ expanded_at: new Date() })
-      articles += jobs.length
-    }
-    return { ranges: ranges.length, articles }
-  }
-
-  /** Reads one body of the file; expands small files into the body queue. Returns the new status. */
-  async process(file: any): Promise<string> {
-    const update: Record<string, unknown> = { processed_at: new Date(), error: null }
-    try {
-      const conn = await NntpService.acquire()
-      let failed = false
-      let body: Buffer
-      try {
-        body = await NntpService.fetchBody(conn, file.first_message_id)
-      } catch (error) {
-        failed = true
-        throw error
-      } finally {
-        NntpService.release(conn, failed)
-      }
-
-      const meta = parseYencMeta(body)
-      const totalParts = inferTotalParts(meta)
-      Object.assign(update, {
-        yenc_name: meta.begin.name,
-        size: meta.begin.size,
-        total_parts: totalParts,
-        sampled_part: meta.begin.part ?? 1,
-      })
-      if ((meta.begin.part ?? 1) === 1) {
-        const data = decodeSegment(body, { lenientSize: true, verifyCrc: false }).data
-        update.content_type = sniffContentType(data)
-        const rar = parseRarHeaders(data)
-        if (rar) update.encrypted = rar.headersEncrypted || rar.entries.some((e) => e.encrypted)
-      }
-
-      update.status = 'sampled'
-      if (totalParts <= indexerConfig.deferredExpandMaxParts && update.encrypted !== true) {
-        const queued = await this.expand(file)
-        if (queued > 0) update.status = 'expanded'
-      }
-    } catch (error: any) {
-      update.status = 'failed'
-      update.error = String(error.message).slice(0, 500)
-    }
-    await db.from('deferred_files').where('id', file.id).update(update)
-    return update.status as string
-  }
-
-  /** Finds the file's articles again (XOVER over its range) and queues their bodies. */
-  private async expand(file: any): Promise<number> {
-    const first = Number(file.first_article)
-    const last = Number(file.last_article)
-    if (last - first > EXPAND_MAX_RANGE) return 0
-
-    const conn = await NntpService.acquire()
-    let failed = false
-    let overviews: [number, OverviewHeader][]
-    try {
-      await conn.group(file.group_name)
-      overviews = (await conn.xover(String(first), String(last))).overviews
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    const jobs = overviews
-      .filter(([, header]) => header && deferredName(header.subject) === file.name)
-      .map(([articleNumber, header]) => bodyJob(header, file.group_name, articleNumber))
-    if (jobs.length > 0) await QueueService.bodyQueue.addBulk(jobs)
-    return jobs.length
-  }
-}
-
-export default new DeferredService()

app/services/FetchRangeService.ts

@@ -1,115 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import QueueService from '#services/QueueService'
-
-/** A failed range is retried after this long... */
-const REQUEUE_AFTER_MS = 5 * 60 * 1000
-/** ...at most this many times; after that it stays visible as a gap. */
-const MAX_REQUEUES = 5
-/** A queued range without a job in Redis (e.g. Redis was flushed) is re-queued after this long. */
-const LOST_AFTER_MS = 60 * 60 * 1000
-
-interface Group {
-  id: number
-  name: string
-}
-
-/**
- * Tracks the article ranges given to worker:fetch. A range only counts as
- * indexed once the worker reports it done, so failed or lost fetch jobs are
- * retried instead of silently skipped. See CONCEPT.md (D18).
- */
-class FetchRangeService {
-  async queue(group: Group, startId: bigint, endId: bigint) {
-    const now = new Date()
-    const [row] = await db
-      .table('fetch_ranges')
-      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now })
-      .returning(['id', 'requeues'])
-    await this.addJob(group.name, Number(row.id), 0, startId, endId)
-  }
-
-  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) {
-    await QueueService.nntpFetchQueue.add(
-      'fetch-headers',
-      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId },
-      // The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
-      { jobId: this.jobId(rangeId, requeues) }
-    )
-  }
-
-  private jobId(rangeId: number, requeues: number) {
-    return `range-${rangeId}-${requeues}`
-  }
-
-  async markDone(rangeId: number) {
-    await db.from('fetch_ranges').where('id', rangeId).update({ status: 'done', error: null, updated_at: new Date() })
-  }
-
-  async markFailed(rangeId: number, error: string) {
-    await db.from('fetch_ranges').where('id', rangeId).update({ status: 'failed', error, updated_at: new Date() })
-  }
-
-  /** Re-queues failed ranges, and queued ranges whose job no longer exists. */
-  async requeue(group: Group): Promise<number> {
-    const now = Date.now()
-    const candidates = await db
-      .from('fetch_ranges')
-      .where('group_id', group.id)
-      .where('requeues', '<', MAX_REQUEUES)
-      .where((q) =>
-        q
-          .where((f) => f.where('status', 'failed').where('updated_at', '<', new Date(now - REQUEUE_AFTER_MS)))
-          .orWhere((l) => l.where('status', 'queued').where('updated_at', '<', new Date(now - LOST_AFTER_MS)))
-      )
-      .orderBy('start_id')
-
-    let requeued = 0
-    for (const range of candidates) {
-      if (range.status === 'queued') {
-        const job = await QueueService.nntpFetchQueue.getJob(this.jobId(Number(range.id), range.requeues))
-        const state = job ? await job.getState() : 'missing'
-        if (state !== 'missing' && state !== 'failed' && state !== 'unknown') continue
-      }
-      await db
-        .from('fetch_ranges')
-        .where('id', range.id)
-        .update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
-      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id))
-      requeued++
-      logger.warn(
-        `Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
-      )
-    }
-    return requeued
-  }
-
-  /**
-   * Moves `groups.last_indexed_id` up to just below the first range that is
-   * not done yet, and forgets done ranges. Returns the new watermark.
-   */
-  async advanceWatermark(group: Group): Promise<bigint> {
-    const { rows } = await db.rawQuery(
-      `UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
-         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'),
-         g.last_queued_id))
-       WHERE g.id = ? RETURNING last_indexed_id`,
-      [group.id]
-    )
-    await db.from('fetch_ranges').where('group_id', group.id).where('status', 'done').delete()
-    return BigInt(rows[0].last_indexed_id)
-  }
-
-  /** Ranges that failed for good (for monitoring). */
-  async abandoned(group: Group) {
-    return db
-      .from('fetch_ranges')
-      .where('group_id', group.id)
-      .where('status', 'failed')
-      .where('requeues', '>=', MAX_REQUEUES)
-      .count('* as count')
-      .first()
-  }
-}
-
-export default new FetchRangeService()

app/services/IngestService.ts

@@ -1,471 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import type { QueryClientContract } from '@adonisjs/lucid/types/database'
-import type { Knex } from 'knex'
-import QueueService from '#services/QueueService'
-import NameService from '#services/NameService'
-import { parseXrefGroups } from '#lib/nntp/xref'
-import { fileKey, parseSubject, weakFileKey } from '#lib/subject/index'
-import { releaseIdentity, type ReleaseIdentity } from '#lib/release/index'
-import { inferTotalParts, type YencMeta } from '#lib/yenc/index'
-
-/** One segment, with the identity of the file it belongs to already resolved. */
-export interface IngestSegment {
-  fileKey: string
-  poster: string
-  /** Subject without segment counter. */
-  subject: string
-  name: string
-  totalParts: number
-  number: number
-  /** Without angle brackets. */
-  messageId: string
-  bytes: number
-  articleNumber?: bigint | number
-  postedAt?: Date
-  groups: string[]
-  /**
-   * The identity comes from the yEnc name only (obfuscated posts). Such names
-   * can be shared by several equal-sized files of one post, so a second
-   * article for the same part number marks the binary as conflicted.
-   */
-  weakIdentity?: boolean
-  /** Release (download) this file belongs to, when it can be determined. */
-  release?: ReleaseIdentity | null
-  /** Exact file size from =ybegin (body-derived segments only). */
-  size?: number
-  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
-  pcrc32?: string
-  partSize?: number
-  fileCrc32?: string
-  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
-  md5_16k?: string
-}
-
-export interface CompletedBinary {
-  id: number
-  fileKey: string
-}
-
-const BINARY_CHUNK = 1000
-/** last_seen_at is only rewritten when it is older than this (it drives GC at 48 h). */
-const TOUCH_MS = 10 * 60 * 1000
-
-type Merge = 'least' | 'coalesce' | 'touch'
-
-/**
- * Inserts rows keyed by `key` and returns their ids. Existing rows are not
- * locked unless something in them changes (`least`: an earlier value,
- * `coalesce`: a value where there was none, `touch`: a timestamp more than
- * TOUCH_MS newer). Parallel fetch jobs see mostly the same rows, and
- * `INSERT … ON CONFLICT DO UPDATE` made them wait on each other for every one.
- */
-async function upsertKeyed(
-  knex: Knex,
-  table: 'binaries' | 'releases',
-  key: string,
-  rows: Record<string, unknown>[],
-  merge: Record<string, [Merge, string]>
-): Promise<Map<string, number>> {
-  const ids = new Map<string, number>()
-  if (rows.length === 0) return ids
-  const inserted: Record<string, any>[] = await knex(table).insert(rows).onConflict(key).ignore().returning(['id', key])
-  for (const row of inserted) ids.set(row[key], Number(row.id))
-
-  const incoming = new Map(rows.filter((r) => !ids.has(r[key] as string)).map((r) => [r[key] as string, r]))
-  if (incoming.size === 0) return ids
-  const columns = Object.keys(merge)
-  const existing: Record<string, any>[] = await knex(table).whereIn(key, [...incoming.keys()]).select('id', key, ...columns)
-
-  const changes: Record<string, unknown>[] = []
-  for (const row of existing) {
-    ids.set(row[key], Number(row.id))
-    const next = incoming.get(row[key])!
-    const changed = columns.some((column) => {
-      const [kind] = merge[column]
-      const current = row[column]
-      const value = next[column]
-      if (value === null || value === undefined) return false
-      if (kind === 'coalesce') return current === null
-      if (kind === 'least') return current === null || value < current
-      return current === null || +(value as Date) - +current > TOUCH_MS
-    })
-    if (changed) changes.push({ id: Number(row.id), ...Object.fromEntries(columns.map((c) => [c, next[c] ?? null])) })
-  }
-  if (changes.length === 0) return ids
-
-  const set = columns
-    .map((c) => {
-      const [kind] = merge[c]
-      const fn = kind === 'least' ? 'least' : kind === 'touch' ? 'greatest' : 'coalesce'
-      return `${c} = ${fn}(t.${c}, v.${c})`
-    })
-    .join(', ')
-  const types = columns.map((c) => `${c} ${merge[c][1]}`).join(', ')
-  changes.sort((a, b) => (a.id as number) - (b.id as number))
-  for (let attempt = 1; ; attempt++) {
-    try {
-      await knex.raw(
-        `UPDATE ${table} t SET ${set}
-         FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, ${types})
-         WHERE t.id = v.id`,
-        [JSON.stringify(changes)]
-      )
-      return ids
-    } catch (error: any) {
-      // Rare now, but two jobs can still update overlapping rows in a different order.
-      if (error.code !== '40P01' || attempt >= 3) throw error
-    }
-  }
-}
-/** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
-export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
-/** A par2 set and its files are posted together; don't match across days. */
-export const PAR2_MATCH_WINDOW_SECONDS = 12 * 60 * 60
-const PART_CHUNK = 2000
-
-function chunks<T>(items: T[], size: number): T[][] {
-  const result: T[][] = []
-  for (let i = 0; i < items.length; i += size) result.push(items.slice(i, i + size))
-  return result
-}
-
-function validDate(date?: Date) {
-  return date && !Number.isNaN(date.getTime()) ? date : null
-}
-
-/** An XOVER overview entry as returned by nntp-js. */
-export type OverviewHeader = Record<string, string>
-
-function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
-  const xrefGroups = parseXrefGroups(header.xref)
-  return {
-    poster: header.from ?? '',
-    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
-    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
-    articleNumber,
-    postedAt: header.date ? new Date(header.date) : undefined,
-    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
-  }
-}
-
-/** Builds a segment from the subject line; null when the subject is obfuscated. */
-export function segmentFromHeader(
-  header: OverviewHeader,
-  groupName: string,
-  articleNumber?: bigint | number
-): IngestSegment | null {
-  const parsed = header.subject ? parseSubject(header.subject) : null
-  if (!parsed || !header['message-id']) return null
-  const base = common(header, groupName, articleNumber)
-  return {
-    ...base,
-    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
-    subject: parsed.base,
-    name: parsed.name,
-    totalParts: parsed.totalSegments,
-    number: parsed.segment,
-    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
-  }
-}
-
-/**
- * Builds a segment for an obfuscated post from the yEnc header in its body.
- * Subject and poster can be random per article, so the identity comes from
- * the yEnc header only (see weakFileKey).
- */
-export function segmentFromYenc(
-  header: OverviewHeader,
-  groupName: string,
-  meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
-  articleNumber?: bigint | number
-): IngestSegment | null {
-  const base = common(header, groupName, articleNumber)
-  const totalParts = inferTotalParts(meta)
-  const number = meta.begin.part ?? 1
-  if (!base.messageId || number < 1 || number > totalParts) return null
-  const subject = `"${meta.begin.name}" yEnc`
-  return {
-    ...base,
-    fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
-    subject,
-    name: meta.begin.name,
-    totalParts,
-    number,
-    weakIdentity: true,
-    release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
-    size: meta.begin.size,
-    // Single-part posts only carry crc32, which is then also the part's CRC.
-    pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
-    partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
-    fileCrc32: meta.end?.crc32,
-  }
-}
-
-/**
- * Stores segments in `binaries`/`parts` and queues every binary that became
- * complete exactly once. Idempotent: ingesting the same segments again is a
- * no-op. See CONCEPT.md (D12).
- */
-class IngestService {
-  async ingest(segments: IngestSegment[]): Promise<CompletedBinary[]> {
-    if (segments.length === 0) return []
-
-    // Short statements, no enclosing transaction: every step is idempotent, and
-    // row locks held for a whole batch made parallel fetch jobs wait on each other.
-    const binaryIds = await (async () => {
-      const knex = db.connection().getWriteClient()
-      const now = new Date()
-      const releaseIds = await this.upsertReleases(knex, segments, now)
-
-      // One row per file_key (ON CONFLICT cannot touch a row twice per statement),
-      // sorted so concurrent workers lock rows in the same order and cannot deadlock.
-      const binaries = new Map<string, Record<string, unknown>>()
-      for (const segment of segments) {
-        const postedAt = validDate(segment.postedAt)
-        const existing = binaries.get(segment.fileKey)
-        if (existing) {
-          if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
-            existing.posted_at = postedAt
-          }
-          continue
-        }
-        binaries.set(segment.fileKey, {
-          file_key: segment.fileKey,
-          poster: segment.poster,
-          subject: segment.subject,
-          name: segment.name,
-          total_parts: segment.totalParts,
-          groups: JSON.stringify(segment.groups),
-          posted_at: postedAt,
-          first_seen_at: now,
-          last_seen_at: now,
-          release_id: segment.release ? releaseIds.get(segment.release.key) : null,
-          size: segment.size ?? null,
-          md5_16k: segment.md5_16k ?? null,
-        })
-      }
-      const binaryRows = [...binaries.values()].sort((a, b) =>
-        (a.file_key as string).localeCompare(b.file_key as string)
-      )
-
-      const idsByKey = new Map<string, number>()
-      for (const chunk of chunks(binaryRows, BINARY_CHUNK)) {
-        const ids = await upsertKeyed(knex, 'binaries', 'file_key', chunk, {
-          last_seen_at: ['touch', 'timestamptz'],
-          posted_at: ['least', 'timestamptz'],
-          release_id: ['coalesce', 'bigint'],
-          md5_16k: ['coalesce', 'varchar'],
-        })
-        for (const [key, id] of ids) idsByKey.set(key, id)
-      }
-
-      const partRows = segments
-        .map((segment) => ({
-          binary_id: idsByKey.get(segment.fileKey)!,
-          number: segment.number,
-          message_id: segment.messageId,
-          bytes: segment.bytes,
-          article_number: segment.articleNumber?.toString() ?? null,
-          pcrc32: segment.pcrc32 ?? null,
-          part_size: segment.partSize ?? null,
-          file_crc32: segment.fileCrc32 ?? null,
-          md5_16k: segment.md5_16k ?? null,
-        }))
-      const weakRows = partRows.filter((_, i) => segments[i].weakIdentity)
-
-      partRows.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
-      for (const chunk of chunks(partRows, PART_CHUNK)) {
-        await knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore()
-      }
-
-      if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
-
-      return [...idsByKey.values()]
-    })()
-
-    // par2 naming needs a size, which only body-derived segments have; for
-    // header batches these queries were most of Postgres' work.
-    const sized = [...new Set(segments.filter((s) => s.size != null).map((s) => s.fileKey))]
-    if (sized.length > 0) {
-      const sizedIds = (await db.from('binaries').whereIn('file_key', sized).select('id')).map((r) => Number(r.id))
-      await this.matchPar2Releases(sizedIds)
-      await NameService.resolve(sizedIds)
-    }
-
-    // After commit, so the last writer of a binary always sees all of its parts.
-    const completed = await this.claimCompleted(binaryIds)
-    await this.enqueue(completed)
-    return completed
-  }
-
-  /**
-   * Creates or touches the releases referenced by `segments` and returns
-   * their ids by release key. Also used by `release:rebuild`.
-   */
-  async upsertReleases(
-    knex: Knex,
-    segments: Pick<IngestSegment, 'release' | 'poster' | 'groups' | 'postedAt' | 'weakIdentity'>[],
-    now = new Date()
-  ): Promise<Map<string, number>> {
-    const releases = new Map<string, Record<string, unknown>>()
-    for (const segment of segments) {
-      const release = segment.release
-      if (!release) continue
-      const postedAt = validDate(segment.postedAt)
-      const existing = releases.get(release.key)
-      if (existing) {
-        if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
-          existing.posted_at = postedAt
-        }
-        // Not every file carries the [i/M] counter (e.g. a par2 index): take it from any that does.
-        if (existing.file_total === null && release.fileTotal) existing.file_total = release.fileTotal
-        continue
-      }
-      releases.set(release.key, {
-        release_key: release.key,
-        name: release.name,
-        // Obfuscating posters randomise the poster, so it says nothing about the release.
-        poster: segment.weakIdentity ? null : segment.poster,
-        file_total: release.fileTotal ?? null,
-        source: release.source,
-        groups: JSON.stringify(segment.groups),
-        posted_at: postedAt,
-        first_seen_at: now,
-        last_seen_at: now,
-      })
-    }
-
-    const ids = new Map<string, number>()
-    const rows = [...releases.values()].sort((a, b) =>
-      (a.release_key as string).localeCompare(b.release_key as string)
-    )
-    for (const chunk of chunks(rows, BINARY_CHUNK)) {
-      const result = await upsertKeyed(knex, 'releases', 'release_key', chunk, {
-        last_seen_at: ['touch', 'timestamptz'],
-        posted_at: ['least', 'timestamptz'],
-        file_total: ['coalesce', 'integer'],
-        source: ['coalesce', 'varchar'],
-      })
-      for (const [key, id] of result) ids.set(key, id)
-    }
-    return ids
-  }
-
-  /**
-   * Assigns body-derived binaries without a release and without a
-   * fingerprint to a known par2 set by size: only when that size is listed
-   * once in the set, the file is large, and exactly one set nearby lists it.
-   * Binaries with a fingerprint are matched exactly by NameService instead.
-   */
-  async matchPar2Releases(binaryIds: number[]) {
-    if (binaryIds.length === 0) return
-    await db.rawQuery(
-      `UPDATE binaries b SET release_id = m.release_id
-       FROM (
-         SELECT b2.id, min(rf.release_id) AS release_id
-         FROM binaries b2
-         JOIN release_files rf ON rf.size = b2.size
-         JOIN releases r ON r.id = rf.release_id
-         WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
-           AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
-           AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
-         GROUP BY b2.id
-         HAVING count(DISTINCT rf.release_id) = 1
-       ) m
-       WHERE b.id = m.id`,
-      [binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
-    )
-  }
-
-  /**
-   * Flags binaries where a stored part has a different message-id and a
-   * different part CRC than the incoming one: two files were merged under
-   * one identity. Identical reposts (same CRC) are ignored.
-   */
-  private async flagConflicts(
-    trx: QueryClientContract,
-    rows: {
-      binary_id: number
-      number: number
-      message_id: string
-      bytes: number
-      article_number: string | null
-      pcrc32: string | null
-      part_size: number | null
-      file_crc32: string | null
-      md5_16k: string | null
-    }[]
-  ) {
-    const bindings = [
-      rows.map((r) => r.binary_id),
-      rows.map((r) => r.number),
-      rows.map((r) => r.message_id),
-      rows.map((r) => r.bytes),
-      rows.map((r) => r.article_number),
-      rows.map((r) => r.pcrc32),
-      rows.map((r) => r.part_size),
-      rows.map((r) => r.file_crc32),
-      rows.map((r) => r.md5_16k),
-    ]
-    // Keep the extra articles: SplitService separates the volumes from them.
-    const conflicts = await trx.rawQuery(
-      `INSERT INTO part_conflicts (binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
-       SELECT v.binary_id, v.number, v.message_id, v.bytes, v.article_number, v.pcrc32, v.part_size, v.file_crc32, v.md5_16k
-       FROM unnest(?::bigint[], ?::int[], ?::text[], ?::int[], ?::bigint[], ?::text[], ?::int[], ?::text[], ?::text[])
-         AS v(binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
-       JOIN parts p ON p.binary_id = v.binary_id AND p.number = v.number
-       WHERE p.message_id <> v.message_id
-         -- Same part CRC: the same data posted again (a repost), not another file.
-         AND (p.pcrc32 IS NULL OR v.pcrc32 IS NULL OR p.pcrc32 <> v.pcrc32)
-       ON CONFLICT DO NOTHING
-       RETURNING binary_id`,
-      bindings as any
-    )
-    const binaryIds: number[] = [
-      ...new Set<number>(conflicts.rows.map((r: { binary_id: string }) => Number(r.binary_id))),
-    ]
-    if (binaryIds.length === 0) return
-    const result = await trx.rawQuery(
-      `UPDATE binaries SET conflicted_at = now()
-       WHERE conflicted_at IS NULL AND id = ANY(?) RETURNING name`,
-      [binaryIds]
-    )
-    for (const row of result.rows) {
-      logger.warn(`Binary "${row.name}" has conflicting parts; it will be split into volumes.`)
-    }
-  }
-
-  /**
-   * Marks complete binaries among `binaryIds` as claimed. The single UPDATE
-   * locks each row, so concurrent callers never claim the same binary twice.
-   */
-  async claimCompleted(binaryIds: number[]): Promise<CompletedBinary[]> {
-    if (binaryIds.length === 0) return []
-    const result = await db.rawQuery(
-      `UPDATE binaries b SET completed_at = now()
-       WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
-         AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) >= b.total_parts
-       RETURNING b.id, b.file_key`,
-      [binaryIds]
-    )
-    return result.rows.map((row: { id: string; file_key: string }) => ({
-      id: Number(row.id),
-      fileKey: row.file_key,
-    }))
-  }
-
-  async enqueue(completed: CompletedBinary[]) {
-    if (completed.length === 0) return
-    // jobId = file_key: BullMQ ignores an add() for an id that already exists.
-    await QueueService.fileQueue.addBulk(
-      completed.map(({ id, fileKey }) => ({
-        name: 'process-file',
-        data: { binaryId: id },
-        opts: { jobId: fileKey },
-      }))
-    )
-  }
-}
-
-export default new IngestService()

app/services/NameService.ts

@@ -1,128 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import { looksRandom } from '#services/ArchiveService'
-
-/**
- * A release that is nothing more than one randomly named file (its own
- * filename stem). Such a file moves into the par2 set that names it exactly;
- * real groupings (par2, RAR, NZB, spot, subject, post id) are left alone.
- */
-export const WEAK_RELEASE_SQL = (column: string) =>
-  `EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem')
-     AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))`
-
-/**
- * Whether `candidate` (e.g. from a par2 listing) should replace `current`:
- * the current name is random, or a single token without separators
- * (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the
- * candidate reads like a release name.
- */
-export function isBetterName(current: string, candidate: string): boolean {
-  if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false
-  return looksRandom(current) || !/[._ -]/.test(current)
-}
-
-/** Smaller files (nfo, sfv) collide too easily on size alone. */
-const MIN_SIZE_ONLY = 64 * 1024
-
-/**
- * Gives binaries (and their files) the real filename that a par2 set lists
- * for them. See CONCEPT.md (D21).
- * - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This
- *   also puts the binary in that par2 set's release.
- * - Size only, when the fingerprint is unknown: within the binary's own
- *   release, only when exactly one listed file has that size.
- */
-class NameService {
-  async resolve(binaryIds: number[]): Promise<number> {
-    if (binaryIds.length === 0) return 0
-    const exact = await db.rawQuery(
-      `UPDATE binaries b SET real_name = m.name,
-         release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END
-       FROM (
-         SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id
-         FROM binaries b2
-         JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k
-         WHERE b2.id = ANY(?)
-         ORDER BY b2.id, rf.release_id DESC
-       ) m
-       WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL
-         OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`,
-      [binaryIds]
-    )
-    const bySize = await db.rawQuery(
-      `UPDATE binaries b SET real_name = m.name
-       FROM (
-         SELECT b2.id, min(rf.name) AS name
-         FROM binaries b2
-         JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size
-         WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
-         GROUP BY b2.id
-         HAVING count(DISTINCT rf.name) = 1
-       ) m
-       WHERE b.id = m.id`,
-      [binaryIds, MIN_SIZE_ONLY]
-    )
-    await db.rawQuery(
-      `UPDATE files f SET real_name = b.real_name,
-         release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END
-       FROM binaries b
-       WHERE b.id = ANY(?) AND b.file_id = f.id
-         AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL)
-           OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`,
-      [binaryIds]
-    )
-    return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0)
-  }
-
-  /**
-   * When a par2 set arrives: the binaries it can name, i.e. those matching
-   * one of its entries exactly (size + first-16-KiB MD5) and those in its
-   * release. Selecting every binary of the listed sizes instead picked up
-   * hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)).
-   */
-  async resolveForRelease(releaseId: number): Promise<number> {
-    const { rows } = await db.rawQuery(
-      `SELECT b.id FROM release_files rf
-       JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size
-       WHERE rf.release_id = ?
-       UNION
-       SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`,
-      [releaseId, releaseId]
-    )
-    return this.resolve(rows.map((r: { id: string }) => Number(r.id)))
-  }
-}
-
-/**
- * Repair for data from before files moved into the par2 set that names them
- * (used by `release:rename`): regroups exact matches out of random-named
- * single-file releases, then renames par2 releases that still carry a
- * random name after their listing. No downloads.
- */
-export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> {
-  const { rows } = await db.rawQuery(
-    `SELECT DISTINCT b.id FROM binaries b
-     JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size
-     WHERE b.release_id IS DISTINCT FROM rf.release_id
-       AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})`
-  )
-  const ids = rows.map((r: { id: string }) => Number(r.id))
-  for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000))
-
-  const { par2ReleaseName } = await import('#lib/release/index')
-  const listings = await db.rawQuery(
-    `SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
-     JOIN release_files rf ON rf.release_id = r.id GROUP BY r.id, r.name`
-  )
-  let renamed = 0
-  for (const release of listings.rows) {
-    const better = par2ReleaseName(release.names)
-    if (!better || better === release.name || !isBetterName(release.name, better)) continue
-    await db.from('releases').where('id', release.id).update({ name: better })
-    renamed++
-  }
-  return { regrouped: ids.length, renamed }
-}
-
-const nameService = new NameService()
-export default nameService

app/services/NntpService.ts

@@ -1,119 +0,0 @@
-import nntpConfig from '#config/nntp'
-import { NNTP } from "nntp-js";
-import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
-
-class NntpService {
-  private readonly poolSize: number;
-  private allConnections: Set<any>;
-  private idleConnections: any[];
-  private waiters: ((conn: any) => void)[];
-  private createdCount: number;
-
-  constructor(poolSize = 10) {
-    this.poolSize = poolSize;
-    this.allConnections = new Set();
-    this.idleConnections = [];
-    this.waiters = [];
-    this.createdCount = 0;
-  }
-
-  private async _createConnection() {
-    const conn = new NNTP(nntpConfig.host, nntpConfig.port);
-    try {
-      await conn.connect();
-      if (nntpConfig.user) {
-        await conn.login(nntpConfig.user, nntpConfig.password?.release());
-      }
-    } catch (error) {
-      // Free the slot, otherwise a failed connect shrinks the pool for good.
-      this.createdCount--;
-      this._destroy(conn);
-      throw error;
-    }
-    this.allConnections.add(conn);
-    return conn;
-  }
-
-  public async acquire() {
-    if (this.idleConnections.length > 0) {
-      return this.idleConnections.pop();
-    }
-
-    if (this.createdCount < this.poolSize) {
-      this.createdCount++;
-      console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`);
-      return this._createConnection();
-    }
-
-    return new Promise(resolve => this.waiters.push(resolve));
-  }
-
-  /**
-   * Returns a connection to the pool. Pass `failed` when a command on it
-   * threw: the connection may be dead (e.g. closed by the server while idle),
-   * so it is discarded instead of being handed to the next job.
-   */
-  public release(conn: any, failed = false) {
-    if (failed) {
-      this.discard(conn);
-      return;
-    }
-    const resolve = this.waiters.shift();
-    if (resolve) {
-      resolve(conn);
-    } else {
-      this.idleConnections.push(conn);
-    }
-  }
-
-  /** Closes a broken connection and frees its slot for a new one. */
-  public discard(conn: any) {
-    this.allConnections.delete(conn);
-    this.createdCount--;
-    this._destroy(conn);
-
-    // Someone is waiting for a slot: open a fresh connection for them.
-    const resolve = this.waiters.shift();
-    if (resolve) {
-      this.createdCount++;
-      this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
-    }
-  }
-
-  private _destroy(conn: any) {
-    try {
-      conn.sock?.destroy();
-    } catch {
-      // already closed
-    }
-  }
-
-  /** Selects a group and returns its correctly parsed article range. */
-  public async selectGroup(conn: any, name: string): Promise<GroupInfo> {
-    const { resp } = await conn.group(name)
-    return parseGroupResponse(resp)
-  }
-
-  /** Fetches a raw (still dot-stuffed) article body as a Buffer. */
-  public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
-    const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
-    const { data } = await conn.body(id)
-    return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
-  }
-
-  public async shutdown() {
-    console.log('Shutting down all connections in the pool.');
-    // A connection the server already closed may never answer QUIT.
-    const quit = (conn: any) =>
-      Promise.race([conn.quit(), new Promise((resolve) => setTimeout(resolve, 2000).unref())])
-        .catch(() => {})
-        .finally(() => this._destroy(conn));
-    await Promise.all([...this.allConnections].map(quit));
-    this.allConnections.clear();
-    this.idleConnections.length = 0;
-    this.waiters.length = 0;
-    this.createdCount = 0;
-  }
-}
-
-export default new NntpService(nntpConfig.poolSize);

app/services/NzbImportService.ts

@@ -1,168 +0,0 @@
-import { createHash } from 'node:crypto'
-import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import IngestService from '#services/IngestService'
-import { type ReleaseIdentity } from '#lib/release/index'
-import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
-import { parseSubject } from '#lib/subject/index'
-import { fileStem } from '#lib/release/index'
-import { looksRandom } from '#services/ArchiveService'
-
-export interface NzbImportResult {
-  releaseId: number
-  releaseName: string
-  listed: number
-  known: number
-  added: number
-  password: boolean
-}
-
-/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
-export function nzbFileName(subject: string): string {
-  const parsed = parseSubject(subject)
-  const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
-  return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
-}
-
-export interface NzbImportOptions {
-  /**
-   * For NZBs inside an archive: the entry's name. It keeps several NZBs of
-   * one archive apart and names their release when the NZB has no name.
-   */
-  entryName?: string
-  /** Move the posted file into the release the NZB describes (default true). */
-  moveContainer?: boolean
-  /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
-  name?: string
-  /** Release source for new releases (default `nzb`). */
-  source?: ReleaseIdentity['source']
-}
-
-/**
- * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
- * Files it lists that are already known (by first segment) keep their
- * release, which the NZB then joins; the others are added as files, so posts
- * spread over groups we do not index still become complete.
- */
-class NzbImportService {
-  async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
-    const nzb = parseNzb(xml)
-    if (!nzb || nzb.files.length === 0) return null
-
-    const entries = nzb.files.map((f) => ({
-      file: f,
-      name: nzbFileName(f.subject),
-      firstId: f.segments[0].id,
-    }))
-
-    return db.transaction(async (trx) => {
-      const known = await trx
-        .from('files')
-        .whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
-        .select('id', 'release_id', `message_ids`)
-      const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
-
-      // Join the release the listed files are already in, else the posted NZB's own, else a new one.
-      const counts = new Map<number, number>()
-      for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
-      let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
-      const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
-      if (!releaseId) {
-        const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
-        const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
-        const ids = await IngestService.upsertReleases(trx.knexClient, [
-          {
-            release: { key, name: title, source: options.source ?? 'nzb' },
-            poster: nzbFile.poster,
-            groups: nzbFile.groups ?? [],
-            postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
-          },
-        ])
-        releaseId = ids.get(key)!
-      }
-
-      const now = new Date()
-      const missing = entries.filter((e) => !knownIds.has(e.firstId))
-      for (let i = 0; i < missing.length; i += 500) {
-        await trx
-          .knexClient('files')
-          .insert(
-            missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
-              file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
-              filename: name,
-              poster: file.poster || nzbFile.poster,
-              date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
-              parts: file.segments.length,
-              size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
-              message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
-              groups: JSON.stringify(file.groups),
-              release_id: releaseId,
-              created_at: now,
-              updated_at: now,
-            }))
-          )
-          .onConflict('file_key')
-          .ignore()
-      }
-      // The poster's NZB says which files form the post: known files join this
-      // release, also when name rules had split them over several.
-      const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
-      await trx
-        .from('files')
-        .whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
-        .update({ release_id: releaseId })
-      if (previous.length) {
-        await trx.rawQuery(
-          `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
-          [releaseId, releaseId, releaseId]
-        )
-        await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
-      }
-      // The NZB file joins the release it describes, unless it is part of a real release already.
-      const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
-      if (own !== releaseId && options.moveContainer !== false) {
-        const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
-        if (!ownRelease || ownRelease.file_count <= 1) {
-          await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
-          if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
-        }
-      }
-
-      const release = await trx.from('releases').where('id', releaseId).first()
-      const update: Record<string, unknown> = {}
-      if (nzb.meta.password) update.password = nzb.meta.password
-      if (!release.file_total) update.file_total = entries.length
-      if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
-      if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
-
-      const result = {
-        releaseId,
-        releaseName: (update.name as string) ?? release.name,
-        listed: entries.length,
-        known: known.length,
-        added: missing.length,
-        password: Boolean(nzb.meta.password),
-      }
-      logger.info(
-        `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
-          `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
-      )
-      return result
-    })
-  }
-
-  /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
-  private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
-    if (nzb.meta.name) return nzb.meta.name
-    const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
-    if (entry && !looksRandom(entry)) return entry
-    const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
-    if (own && !looksRandom(own)) return own
-    const largest = nzb.files
-      .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
-      .sort((a, b) => b.bytes - a.bytes)[0]
-    return fileStem(largest?.name ?? own)
-  }
-}
-
-export default new NzbImportService()

app/services/Par2Service.ts

@@ -1,165 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import NntpService from '#services/NntpService'
-import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
-import { decodeSegment, YencFile } from '#lib/yenc/index'
-import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
-import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
-import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
-
-/** Par2 index files are small; recovery volumes can be large and are not needed. */
-const MAX_PARTS = 20
-/** Size-only matching below this is too ambiguous. */
-const SIZE_ONLY_MIN = 1024 * 1024
-
-function countBy<T>(values: T[]): Map<T, number> {
-  const counts = new Map<T, number>()
-  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
-  return counts
-}
-
-export interface Par2Result {
-  releaseId: number
-  releaseName: string
-  listed: number
-  matched: number
-}
-
-/**
- * Recognises releases of obfuscated posts from their par2 files: the par2
- * lists the real name and exact size of every file in the set, and
- * body-derived binaries carry their exact size. See CONCEPT.md (D16).
- */
-class Par2Service {
-  /**
-   * Reads a par2 file whose first segment is already decoded, downloading the
-   * remaining parts if there are any. Returns null when it is not par2.
-   */
-  async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
-    if (!isPar2(first)) return null
-    const ids = Object.entries(messageIds)
-      .sort(([a], [b]) => Number(a) - Number(b))
-      .map(([, part]) => part.id)
-    if (ids.length > MAX_PARTS) return null
-    if (ids.length === 1) return parsePar2(first)
-
-    let assembled: YencFile | undefined
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      for (const id of ids) {
-        const segment = decodeSegment(await NntpService.fetchBody(conn, id))
-        assembled ??= YencFile.fromSegment(segment)
-        assembled.add(segment)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-    return parsePar2(assembled!.getBuffer())
-  }
-
-  async attach(binary: any, set: Par2Set): Promise<Par2Result> {
-    const result = await db.transaction(async (trx) => {
-      // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
-      // the set into a release the files already belong to; size alone is
-      // ambiguous across posts (standard volume sizes recur).
-      const exact = await trx
-        .from('binaries')
-        .whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
-        .select('id', 'release_id')
-
-      // Size only, for files without a fingerprint: the size must be listed
-      // once in this set, be large, and belong to exactly one binary nearby.
-      const listedOnce = [...countBy(set.files.map((f) => f.length))]
-        .filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
-        .map(([size]) => size)
-      const bySize = listedOnce.length
-        ? await trx
-            .from('binaries')
-            .whereIn('size', listedOnce)
-            .whereNull('md5_16k')
-            .whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
-              binary.posted_at,
-              PAR2_MATCH_WINDOW_SECONDS,
-            ])
-            .select('id', 'release_id', 'size')
-        : []
-      const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
-      const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
-      const candidates = [...exact, ...fuzzy]
-
-      // Prefer a release the exactly matched files are already in (e.g.
-      // obfuscated volumes grouped by their stem); otherwise the set's own.
-      const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
-      let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
-      const identity = par2ReleaseIdentity(
-        set.recoverySetId,
-        set.files.map((f) => f.name)
-      )
-      if (!releaseId) {
-        const ids = await IngestService.upsertReleases(trx.knexClient, [
-          {
-            release: identity,
-            poster: binary.poster,
-            weakIdentity: true,
-            groups: binary.groups,
-            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
-          },
-        ])
-        releaseId = ids.get(identity.key)!
-      }
-
-      await trx
-        .knexClient('release_files')
-        .insert(
-          set.files.map((f) => ({
-            release_id: releaseId,
-            par2_file_id: f.fileId,
-            name: f.name,
-            size: f.length,
-            md5_16k: f.md5_16k,
-            md5: f.md5,
-            set_id: set.recoverySetId,
-          }))
-        )
-        .onConflict(['release_id', 'par2_file_id'])
-        .ignore()
-
-      const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
-      await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
-      // Exact matches sitting alone in a random-named release (one file of a
-      // season, each posted under its own random name) join the set's release.
-      await trx.rawQuery(
-        `UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
-        [releaseId, exact.map((c) => Number(c.id)), releaseId]
-      )
-      await trx.rawQuery(
-        `UPDATE files f SET release_id = b.release_id FROM binaries b
-         WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
-        [releaseId]
-      )
-
-      // The set may have joined a release named after one random filename.
-      let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
-      const better = par2ReleaseName(set.files.map((f) => f.name))
-      if (releaseName && better && isBetterName(releaseName, better)) {
-        await trx.from('releases').where('id', releaseId).update({ name: better })
-        releaseName = better
-      }
-      logger.info(
-        `Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
-          `${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
-      )
-      return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
-    })
-
-    // Real names for every known file of the set (exact fingerprint, or size within the release).
-    await NameService.resolveForRelease(result.releaseId)
-    return result
-  }
-}
-
-export default new Par2Service()

app/services/QueueService.ts

@@ -1,61 +0,0 @@
-import { Queue, type DefaultJobOptions } from 'bullmq'
-import queueConfig from '#config/queue'
-import gcConfig from '#config/gc'
-
-const defaultJobOptions: DefaultJobOptions = {
-  // Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
-  attempts: 3,
-  backoff: { type: 'exponential', delay: 2000 },
-  removeOnComplete: {
-    age: gcConfig.jobs.completedAgeSeconds,
-    count: gcConfig.jobs.completedMaxCount,
-  },
-  removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
-}
-
-/**
- * Queues are created (and connect to Redis) on first use. Ace imports every
- * command at boot, so eager queues would keep one-off commands from exiting.
- */
-class QueueService {
-  private readonly queues = new Map<string, Queue>()
-
-  private queue(name: string): Queue {
-    let queue = this.queues.get(name)
-    if (!queue) {
-      queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
-      this.queues.set(name, queue)
-    }
-    return queue
-  }
-
-  get nntpFetchQueue() {
-    return this.queue('nntp-fetch-queue')
-  }
-  get fileQueue() {
-    return this.queue('file-queue')
-  }
-  get bodyQueue() {
-    return this.queue('body-queue')
-  }
-  get collectionQueue() {
-    return this.queue('collection-queue')
-  }
-
-  /** Queues no longer used by the pipeline, kept here so GC can empty them. */
-  get legacy(): Queue[] {
-    return [this.queue('header-queue')]
-  }
-
-  get all(): Queue[] {
-    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue]
-  }
-
-  /** Closes the queues that were opened. */
-  async closeAll() {
-    await Promise.all([...this.queues.values()].map((queue) => queue.close()))
-    this.queues.clear()
-  }
-}
-
-export default new QueueService()

app/services/RedisService.ts

@@ -1,25 +0,0 @@
-import { createRequire } from 'module'
-import redisConfig from '#config/redis'
-
-const require = createRequire(import.meta.url)
-const IORedis = require('ioredis')
-
-/**
- * Connects on first use. Ace imports every command at boot, so an eager
- * connection would keep even `node ace list` from exiting.
- */
-class RedisService {
-  private instance?: any
-
-  public get client(): any {
-    this.instance ??= new IORedis(redisConfig.connections.main)
-    return this.instance
-  }
-
-  async close() {
-    if (this.instance) await this.instance.quit()
-    this.instance = undefined
-  }
-}
-
-export default new RedisService()

app/services/SearchService.ts

@@ -1,284 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import { buildNzb } from '#lib/nzb/index'
-import SpotService from '#services/SpotService'
-
-export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
-
-export interface SearchParams {
-  q?: string
-  /** Only releases whose name is readable (not an obfuscated token). */
-  readable?: boolean
-  /** Only releases that have all their files. */
-  complete?: boolean
-  /** Hide password-protected releases. */
-  hideEncrypted?: boolean
-  source?: string
-  poster?: string
-  sort?: SortOrder
-  page?: number
-  perPage?: number
-  /** Newznab: exact category ids (already expanded from parents). */
-  categories?: number[]
-  /** Newznab: leave out releases known to be missing files. */
-  notIncomplete?: boolean
-  maxAgeDays?: number
-  /** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
-  season?: number
-  episode?: number
-  /** Overrides `page` (Newznab uses offsets). */
-  offset?: number
-  /** Leave out releases that are only an encrypted container (nzbe). */
-  hideContainers?: boolean
-  /** Include incomplete releases that consist of par2 files only (hidden by default). */
-  includePar2Only?: boolean
-}
-
-const SORTS: Record<SortOrder, string> = {
-  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
-  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
-  largest: 'r.size DESC, r.id DESC',
-  files: 'r.file_count DESC, r.id DESC',
-  name: 'r.name ASC, r.id ASC',
-}
-
-/**
- * Shown in search: releases with files, and Spotnet listings whose NZB is
- * fetched on first download (CONCEPT.md D29). Matches the partial indexes.
- */
-const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
-
-/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
-const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
-  SELECT 1 FROM files f WHERE f.release_id = r.id
-    AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
-
-/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
-function pattern(word: string): string {
-  const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
-  return `%${escaped.replace(/[._\- ]+/g, '_')}%`
-}
-
-/**
- * Searching and browsing releases for the web interface (CONCEPT.md D22).
- */
-class SearchService {
-  async search(params: SearchParams) {
-    const where: string[] = [
-      VISIBLE_SQL,
-      // Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
-      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`,
-    ]
-    const bindings: unknown[] = []
-
-    const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8)
-    for (const word of words) {
-      const p = pattern(word)
-      // Every word must match the release name, a file name, or the archive contents.
-      // One trigram-indexed lookup per table, not a subquery per release.
-      where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ?
-        OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`)
-      bindings.push(p, p, p, p)
-    }
-    if (params.readable) where.push('is_readable_name(r.name)')
-    if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE')
-    if (params.hideContainers) where.push('r.container IS NULL')
-    // par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
-    if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
-    // Kept current by triggers (migration …860).
-    if (params.complete) where.push('r.complete')
-    if (params.source) {
-      where.push('r.source = ?')
-      bindings.push(params.source)
-    }
-    if (params.poster) {
-      where.push('r.poster ILIKE ?')
-      bindings.push(pattern(params.poster))
-    }
-    if (params.categories?.length) {
-      where.push('r.category = ANY(?)')
-      bindings.push(params.categories)
-    }
-    if (params.notIncomplete) {
-      where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
-    }
-    if (params.maxAgeDays) {
-      where.push(`r.posted_at >= now() - make_interval(days => ?)`)
-      bindings.push(params.maxAgeDays)
-    }
-    if (params.season !== undefined) {
-      const s = params.season
-      const e = params.episode
-      where.push('r.name ~* ?')
-      bindings.push(
-        e !== undefined
-          ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
-          : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
-      )
-    }
-
-    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
-    const page = Math.max(params.page ?? 1, 1)
-    const offset = params.offset ?? (page - 1) * perPage
-    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
-    const whereSql = where.join(' AND ')
-
-    const [{ rows }, { rows: countRows }] = await Promise.all([
-      db.rawQuery(
-        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
-                r.size, r.encrypted, is_readable_name(r.name) AS readable,
-                (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
-                (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
-                   SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
-                   WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
-                jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
-                r.complete, ${PAR2_ONLY_SQL} AS par2_only
-         FROM releases r WHERE ${whereSql}
-         ORDER BY ${order} LIMIT ? OFFSET ?`,
-        [...bindings, perPage, offset]
-      ),
-      db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings),
-    ])
-
-    return {
-      page,
-      perPage,
-      offset,
-      total: Number(countRows[0].total),
-      releases: rows.map((r: any) => ({
-        id: Number(r.id),
-        name: r.name,
-        source: r.source,
-        poster: r.poster,
-        groups: r.groups,
-        postedAt: r.posted_at,
-        fileTotal: r.file_total,
-        fileCount: r.file_count,
-        size: Number(r.size),
-        encrypted: r.encrypted,
-        readable: r.readable,
-        listedFiles: Number(r.listed_files),
-        listedPresent: Number(r.listed_present),
-        archiveEntries: Number(r.archive_entries),
-        complete: r.complete,
-        category: r.category,
-        container: r.container,
-        par2Only: r.par2_only,
-      })),
-    }
-  }
-
-  async release(id: number) {
-    const release = await db.from('releases').where('id', id).first()
-    if (!release) return null
-
-    const [files, listed, pending] = await Promise.all([
-      db
-        .from('files as f')
-        .leftJoin('binaries as b', 'b.file_id', 'f.id')
-        .where('f.release_id', id)
-        .orderByRaw('coalesce(f.real_name, f.filename)')
-        .select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
-      db.rawQuery(
-        `SELECT rf.name, rf.size,
-                EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
-                        WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
-         FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
-        [id]
-      ),
-      db.rawQuery(
-        `SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
-                coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
-         FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
-        [id]
-      ),
-    ])
-
-    const spot = release.source === 'spot'
-      ? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first()
-      : null
-    return {
-      id: Number(release.id),
-      name: release.name,
-      source: release.source,
-      spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null,
-      listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
-      poster: release.poster,
-      groups: release.groups,
-      postedAt: release.posted_at,
-      fileTotal: release.file_total,
-      fileCount: release.file_count,
-      size: Number(release.size),
-      encrypted: release.encrypted,
-      container: release.container,
-      contents: release.contents ?? [],
-      files: files.map((f) => ({
-        id: Number(f.id),
-        name: f.real_name ?? f.filename,
-        postedName: f.filename,
-        realName: f.real_name,
-        size: Number(f.exact_size ?? f.size),
-        parts: f.parts,
-        postedAt: f.date ? new Date(Number(f.date)) : null,
-        poster: f.poster,
-      })),
-      par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
-      pending: {
-        binaries: Number(pending.rows[0].binaries),
-        partsHave: Number(pending.rows[0].parts_have),
-        partsTotal: Number(pending.rows[0].parts_total),
-      },
-    }
-  }
-
-  async nzb(id: number): Promise<{ name: string; xml: string } | null> {
-    let release = await db.from('releases').where('id', id).first()
-    if (!release) return null
-    if (release.source === 'spot' && Number(release.file_count) === 0) {
-      // A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
-      const imported = await SpotService.importListing(id)
-      if (!imported) return null
-      id = imported
-      release = await db.from('releases').where('id', id).first()
-    }
-    const files = await db
-      .from('files')
-      .where('release_id', id)
-      .orderByRaw('coalesce(real_name, filename)')
-      .select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids')
-    const xml = buildNzb(
-      files.map((f) => ({
-        name: f.real_name ?? f.filename,
-        poster: f.poster,
-        date: Number(f.date),
-        groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
-        segments: f.message_ids,
-      })),
-      { title: release.name, password: release.password ?? undefined }
-    )
-    return { name: release.name, xml }
-  }
-
-  async stats() {
-    const { rows } = await db.rawQuery(`
-      SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases,
-             (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases,
-             (SELECT count(*) FROM files) AS files,
-             (SELECT coalesce(sum(size), 0) FROM files) AS size,
-             (SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases,
-             (SELECT max(updated_at) FROM groups) AS last_indexed_at,
-             (SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC)
-                FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`)
-    const r = rows[0]
-    return {
-      releases: Number(r.releases),
-      readableReleases: Number(r.readable_releases),
-      files: Number(r.files),
-      size: Number(r.size),
-      encryptedReleases: Number(r.encrypted_releases),
-      lastIndexedAt: r.last_indexed_at,
-      sources: r.sources ?? [],
-    }
-  }
-}
-
-export default new SearchService()

app/services/Shutdown.ts

@@ -1,45 +0,0 @@
-import { setTimeout as sleep } from 'node:timers/promises'
-
-/** After a signal, give cleanup this long before exiting anyway. */
-const FORCE_EXIT_MS = 25_000
-
-/**
- * Graceful shutdown for the long-running commands. SIGTERM (docker stop) and
- * SIGINT wake up `wait()` and `sleep()`, so workers can finish their active
- * jobs and close their connections before the process exits.
- */
-class Shutdown {
-  private readonly controller = new AbortController()
-
-  constructor() {
-    for (const signal of ['SIGTERM', 'SIGINT'] as const) {
-      process.once(signal, () => {
-        this.controller.abort()
-        setTimeout(() => process.exit(0), FORCE_EXIT_MS).unref()
-      })
-    }
-  }
-
-  get requested(): boolean {
-    return this.controller.signal.aborted
-  }
-
-  /** Resolves when a shutdown signal arrives. */
-  wait(): Promise<void> {
-    if (this.requested) return Promise.resolve()
-    return new Promise((resolve) =>
-      this.controller.signal.addEventListener('abort', () => resolve(), { once: true })
-    )
-  }
-
-  /** Sleeps, but returns early on shutdown. */
-  async sleep(ms: number): Promise<void> {
-    try {
-      await sleep(ms, undefined, { signal: this.controller.signal })
-    } catch {
-      // aborted by a shutdown signal
-    }
-  }
-}
-
-export default new Shutdown()

app/services/SplitService.ts

@@ -1,146 +0,0 @@
-import { createHash } from 'node:crypto'
-import db from '@adonisjs/lucid/services/db'
-import IngestService from '#services/IngestService'
-import NameService from '#services/NameService'
-import { splitVerified } from '#lib/split/index'
-
-/** Conflicted binaries handled per run; the rest follow on the next tick. */
-const BATCH = 200
-
-export interface SplitResult {
-  parents: number
-  volumes: number
-  completed: number
-}
-
-/**
- * Splits conflicted obfuscated binaries (several equal-sized files under one
- * size+parts identity) into one binary per volume. Only volumes whose
- * combined part CRCs match the file CRC of their last part are created, so a
- * split can never produce a corrupt file; the rest waits for more articles.
- * See CONCEPT.md (D20).
- */
-class SplitService {
-  async splitPending(): Promise<SplitResult> {
-    const parents = await db
-      .from('binaries')
-      .whereNotNull('conflicted_at')
-      .whereNull('parent_binary_id')
-      // split_at = 'infinity' marks binaries that cannot be split (see migration …810).
-      .where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
-      // Most recently active first: those are the posts still coming in.
-      .orderBy('last_seen_at', 'desc')
-      .limit(BATCH)
-
-    const volumeIds: number[] = []
-    for (const parent of parents) {
-      volumeIds.push(...(await this.split(parent)))
-    }
-
-    // Former conflicts that turned out to be reposts (unflagged, keep their
-    // split_at): no new article may arrive to trigger their completion.
-    const unflagged = await db
-      .from('binaries')
-      .whereNull('conflicted_at')
-      .whereNull('completed_at')
-      .whereNull('parent_binary_id')
-      .whereNotNull('split_at')
-      .whereRaw("split_at <> 'infinity'")
-      .limit(BATCH)
-      .select('id')
-    volumeIds.push(...unflagged.map((b) => Number(b.id)))
-
-    await IngestService.matchPar2Releases(volumeIds)
-    await NameService.resolve(volumeIds)
-    const completed = await IngestService.claimCompleted(volumeIds)
-    await IngestService.enqueue(completed)
-    return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
-  }
-
-  /** Creates the newly verified volumes of one conflicted binary; returns their ids. */
-  private async split(parent: any): Promise<number[]> {
-    return db.transaction(async (trx) => {
-      // Unverified volumes from before verification existed can never complete.
-      await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()
-
-      const { rows } = await trx.rawQuery(
-        `WITH taken AS (
-           SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
-           WHERE v.parent_binary_id = ?
-         ), candidates AS (
-           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
-           UNION ALL
-           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
-         )
-         SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
-        [parent.id, parent.id, parent.id]
-      )
-      const volumes = splitVerified(
-        rows.map((r: any) => ({
-          number: r.number,
-          messageId: r.message_id,
-          bytes: r.bytes,
-          articleNumber: r.article_number === null ? null : Number(r.article_number),
-          pcrc32: r.pcrc32,
-          partSize: r.part_size,
-          fileCrc32: r.file_crc32,
-        })),
-        { totalParts: parent.total_parts }
-      )
-
-      const now = new Date()
-      const ids: number[] = []
-      for (const articles of volumes) {
-        // A verified volume is fixed, so its first article identifies it.
-        const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
-        const inserted = await trx
-          .knexClient('binaries')
-          .insert({
-            file_key: fileKey,
-            poster: parent.poster,
-            subject: parent.subject,
-            name: parent.name,
-            total_parts: parent.total_parts,
-            groups: JSON.stringify(parent.groups),
-            posted_at: parent.posted_at,
-            size: parent.size,
-            release_id: parent.release_id,
-            parent_binary_id: parent.id,
-            first_seen_at: now,
-            last_seen_at: now,
-          })
-          .onConflict('file_key')
-          .ignore()
-          .returning('id')
-        if (inserted.length === 0) continue
-        const id = Number(inserted[0].id)
-        await trx.knexClient('parts').insert(
-          articles.map((a) => ({
-            binary_id: id,
-            number: a.number,
-            message_id: a.messageId,
-            bytes: a.bytes,
-            article_number: a.articleNumber,
-            pcrc32: a.pcrc32,
-            part_size: a.partSize,
-            file_crc32: a.fileCrc32,
-          }))
-        )
-        // The fingerprint of a volume is that of its part 1 article.
-        await trx.rawQuery(
-          `UPDATE binaries SET md5_16k = (
-             SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
-             UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
-           WHERE id = ?`,
-          [parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
-        )
-        ids.push(id)
-      }
-
-      await trx.from('binaries').where('id', parent.id).update({ split_at: now })
-      return ids
-    })
-  }
-}
-
-export default new SplitService()

app/services/SpotService.ts

@@ -1,172 +0,0 @@
-import db from '@adonisjs/lucid/services/db'
-import logger from '@adonisjs/core/services/logger'
-import indexerConfig from '#config/indexer'
-import NntpService from '#services/NntpService'
-import NzbImportService from '#services/NzbImportService'
-import { type OverviewHeader } from '#services/IngestService'
-import { parseNzb } from '#lib/nzb/index'
-import {
-  moderationTarget,
-  spotterId,
-  decodeSpotBinary,
-  parseSpotArticleHeaders,
-  parseSpotHeader,
-  parseSpotXml,
-  verifySpotHeader,
-  type SpotHeader,
-} from '#lib/spotnet/index'
-
-/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
-const MAX_NZB_SEGMENTS = 20
-/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
-const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
-/** A claimed spot whose worker died is taken again after this long. */
-const RECLAIM_AFTER = '1 hour'
-
-export function isSpotGroup(name: string): boolean {
-  return indexerConfig.spotGroups.includes(name)
-}
-
-/**
- * Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
- * header holds XML with the message ids of a compressed NZB, which is
- * imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
- * (nyuu, ngPost, the skipped multi-group uploader).
- */
-class SpotService {
-  /** Queues the spots among a batch of one spot group's headers. Returns how many. */
-  async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
-    const now = new Date()
-    const rows = []
-    for (const [articleNumber, header] of overviews) {
-      if (!header?.from || !header['message-id']) continue
-      const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
-      if (!spot?.title) continue
-      rows.push({
-        message_id: spot.messageId,
-        group_name: groupName,
-        article_number: Number(articleNumber),
-        title: spot.title,
-        tag: spot.tag || null,
-        poster: spot.poster,
-        category: spot.category,
-        subcats: JSON.stringify(spot.subcats),
-        size: spot.size || null,
-        posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
-        key_id: spot.keyId,
-        spotter_id: spotterId(spot.selfSignedPubKey) || null,
-        created_at: now,
-      })
-    }
-    if (rows.length === 0) return 0
-    await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
-    return rows.length
-  }
-
-  /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
-  async claim(limit: number): Promise<any[]> {
-    const { rows } = await db.rawQuery(
-      `UPDATE spots SET status = 'processing', processed_at = now()
-       WHERE message_id IN (
-         SELECT message_id FROM spots
-         WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
-         ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
-       RETURNING *`,
-      [limit]
-    )
-    return rows
-  }
-
-  /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
-  async process(spot: any): Promise<string> {
-    const update: Record<string, unknown> = { processed_at: new Date(), error: null }
-    try {
-      const raw = await this.fetch(`HEAD <${spot.message_id}>`)
-      const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
-      // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
-      const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
-      if (!header || !verifySpotHeader(header)) {
-        update.status = 'unverified'
-      } else if (moderationTarget(header)) {
-        update.status = 'moderation'
-        await this.moderate(header, moderationTarget(header)!)
-      } else {
-        const parsed = parseSpotXml(xml)
-        update.title = header.title
-        update.poster = header.poster
-        update.spotter_id = spotterId(header.selfSignedPubKey) || null
-        if (parsed?.website) update.website = parsed.website
-        const segments = parsed?.nzbSegments ?? []
-        if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
-          update.status = 'no-nzb'
-        } else {
-          const bodies = []
-          for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
-          const nzbXml = decodeSpotBinary(bodies).toString('utf8')
-          const nzb = parseNzb(nzbXml)
-          if (!nzb || nzb.files.length === 0) {
-            update.status = 'no-nzb'
-          } else {
-            const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
-            const result = await NzbImportService.import(
-              {
-                id: null,
-                filename: `${header.title}.nzb`,
-                poster: header.poster,
-                groups,
-                date: header.stamp * 1000,
-                message_ids: { '1': { id: spot.message_id } },
-              },
-              nzbXml,
-              { name: header.title, source: 'spot', moveContainer: false }
-            )
-            update.status = result ? 'imported' : 'no-nzb'
-            if (result) update.release_id = result.releaseId
-          }
-        }
-      }
-    } catch (error: any) {
-      update.status = 'failed'
-      update.error = String(error.message).slice(0, 500)
-      logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
-    }
-    await db.from('spots').where('message_id', spot.message_id).update(update)
-    return update.status as string
-  }
-
-  /**
-   * Applies a delete/dispose/remove: the target spot is marked `removed`
-   * (its release is then hidden from search). As in Spotweb, a personal
-   * dispose must come from the same spotter within 5 days; a moderator's
-   * (signed with Spotnet's key, no personal key) always counts.
-   */
-  private async moderate(header: SpotHeader, targetId: string) {
-    const target = await db.from('spots').where('message_id', targetId).first()
-    if (!target) return
-    const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
-    if (personal) {
-      const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
-      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
-    }
-    await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
-    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
-  }
-
-  /** One HEAD or BODY, as raw bytes. */
-  private async fetch(command: string): Promise<Buffer> {
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      const { data } = await conn._artcmd(command)
-      return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
-    } catch (error: any) {
-      // 430 (no such article) is an answer, not a broken connection.
-      failed = !/^4\d\d/.test(String(error.message))
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-  }
-}
-
-export default new SpotService()

app/services/access_service.ts

@@ -0,0 +1,171 @@
+import { randomBytes } from 'node:crypto'
+import { DateTime } from 'luxon'
+import db from '@adonisjs/lucid/services/db'
+import { Exception } from '@adonisjs/core/exceptions'
+import type { HttpContext } from '@adonisjs/core/http'
+import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
+import User from '#models/user'
+import { ACCESS, SIGNUP, TIERS, type TierLimits } from '#config/access'
+
+export type Use = 'api' | 'grab'
+
+/** Newznab's error codes for access (the catalog answers with the HTTP status). */
+const STATUS = { 100: 401, 101: 403, 102: 403, 500: 429, 501: 429 } as const
+
+/** Why a request is refused; the catalog's answer (Newznab catches it for its own). */
+export class AccessDenied extends Exception {
+  constructor(
+    readonly newznabCode: keyof typeof STATUS,
+    message: string
+  ) {
+    super(message, { status: STATUS[newznabCode], code: 'E_ACCESS_DENIED' })
+  }
+
+  async handle(error: this, ctx: HttpContext) {
+    ctx.response.status(error.status).send({ error: error.message, code: error.newznabCode })
+  }
+}
+
+export interface Usage {
+  apiHits: number
+  grabs: number
+}
+
+/** A caller let in: who, their tier as it is now, and what they used today. */
+export interface Grant {
+  user: User
+  tier: string
+  limits: TierLimits
+  usage: Usage
+}
+
+const COLUMNS: Record<Use, string> = { api: 'api_hits', grab: 'grabs' }
+const TODAY = `(now() AT TIME ZONE 'UTC')::date`
+const DAY_MS = 86_400_000
+
+/** Accounts, tiers and their daily limits (config/access.ts). */
+export default class AccessService {
+  /** From config/access.ts; tests switch it. */
+  static mode: 'open' | 'accounts' = ACCESS
+  static signup: 'open' | 'invite' | 'closed' = SIGNUP
+
+  static get accounts() {
+    return this.mode === 'accounts'
+  }
+
+  static newKey() {
+    return randomBytes(16).toString('hex')
+  }
+
+  /** The user's tier as it is now: one that ran out (or isn't configured) is free. */
+  static tierOf(user: User): { tier: string; limits: TierLimits } {
+    const expired = user.tierUntil !== null && user.tierUntil.toMillis() <= Date.now()
+    const tier = !expired && TIERS[user.tier] ? user.tier : 'free'
+    return { tier, limits: TIERS[tier] }
+  }
+
+  /** The user a request names: the web interface's token, else an apikey (Newznab, the catalog's NZB links). */
+  static async caller(ctx: HttpContext, apikey?: unknown): Promise<User | null> {
+    const guard = ctx.auth.use('api')
+    if (guard.isAuthenticated || (await guard.check())) return guard.user!
+    if (typeof apikey === 'string' && /^[0-9a-f]{32}$/.test(apikey)) return User.findBy('apiKey', apikey)
+    return null
+  }
+
+  /** Lets a request in (counting nothing yet); null when access is open. */
+  static async authorize(ctx: HttpContext, apikey?: unknown): Promise<Grant | null> {
+    if (!this.accounts) return null
+    const user = await this.caller(ctx, apikey)
+    if (!user) throw new AccessDenied(100, 'Incorrect user credentials')
+    if (user.suspended) throw new AccessDenied(101, 'Account suspended')
+    return { user, ...this.tierOf(user), usage: await this.usage(user.id) }
+  }
+
+  /** Counts one use against the grant's tier, or refuses it once the day's limit is reached. */
+  static async charge(grant: Grant | null, use: Use) {
+    if (!grant) return
+    const usage = await this.count(grant.user.id, use, use === 'api' ? grant.limits.apiHits : grant.limits.grabs)
+    if (!usage) throw use === 'api' ? new AccessDenied(500, 'Request limit reached') : new AccessDenied(501, 'Download limit reached')
+    grant.usage = usage
+  }
+
+  /**
+   * Adds one use to today's row unless the limit is reached (null then): one
+   * statement, so concurrent requests can't both take the last one.
+   */
+  static async count(userId: number, use: Use, limit: number | null): Promise<Usage | null> {
+    if (limit !== null && limit <= 0) return null
+    const column = COLUMNS[use]
+    const { rows } = await db.rawQuery(
+      `INSERT INTO usage (user_id, day, ${column}) VALUES (?, ${TODAY}, 1)
+       ON CONFLICT (user_id, day) DO UPDATE SET ${column} = usage.${column} + 1
+       ${limit === null ? '' : `WHERE usage.${column} < ?`}
+       RETURNING api_hits, grabs`,
+      limit === null ? [userId] : [userId, limit]
+    )
+    return rows[0] ? { apiHits: rows[0].api_hits, grabs: rows[0].grabs } : null
+  }
+
+  static async usage(userId: number): Promise<Usage> {
+    const { rows } = await db.rawQuery(`SELECT api_hits, grabs FROM usage WHERE user_id = ? AND day = ${TODAY}`, [userId])
+    return { apiHits: rows[0]?.api_hits ?? 0, grabs: rows[0]?.grabs ?? 0 }
+  }
+
+  /** A search's age bound: the asked one within the tier's retention. */
+  static maxAgeDays(grant: Grant | null, requested?: number): number | undefined {
+    const retention = grant?.limits.retentionDays
+    if (!retention) return requested
+    return requested ? Math.min(requested, retention) : retention
+  }
+
+  /** Refuses a release posted before the tier's retention (one without a date is let through). */
+  static checkPosted(grant: Grant | null, postedAt: Date | string | null) {
+    const retention = grant?.limits.retentionDays
+    if (!retention || !postedAt) return
+    if (new Date(postedAt).getTime() < Date.now() - retention * DAY_MS) {
+      throw new AccessDenied(102, `Older than your tier's retention (${retention} days)`)
+    }
+  }
+
+  /** Whether a release exists, refusing it when it's older than the tier sees; asked before an NZB is counted. */
+  static async checkRelease(grant: Grant | null, id: number): Promise<boolean> {
+    const release = await db.from('releases').where('id', id).select('posted_at').first()
+    if (!release) return false
+    this.checkPosted(grant, release.posted_at)
+    return true
+  }
+
+  /** Takes an unused invite for a new user: what it gives, or null when it's unknown or used. */
+  static async redeem(code: string, userId: number, trx: TransactionClientContract) {
+    const { rows } = await trx.rawQuery(
+      `UPDATE invites SET used_by = ?, used_at = now() WHERE code = ? AND used_by IS NULL AND used_at IS NULL RETURNING tier, tier_days`,
+      [userId, code]
+    )
+    return (rows[0] as { tier: string | null; tier_days: number | null } | undefined) ?? null
+  }
+
+  /**
+   * Gives a user a tier for `days` (null: without end). The same tier still
+   * running is extended from where it ends, so a renewal paid early loses nothing.
+   */
+  static grantTier(user: User, tier: string, days: number | null) {
+    if (!TIERS[tier]) throw new Error(`No such tier: ${tier} (config/access.ts has ${Object.keys(TIERS).join(', ')})`)
+    const now = DateTime.utc()
+    const running = user.tier === tier && user.tierUntil && user.tierUntil > now ? user.tierUntil : now
+    user.tier = tier
+    user.tierUntil = days === null ? null : running.plus({ days })
+  }
+
+  /** What a profile shows of the account's access. */
+  static async summary(user: User) {
+    const { tier, limits } = this.tierOf(user)
+    return {
+      mode: this.mode,
+      tier,
+      tierUntil: tier === user.tier ? user.tierUntil : null,
+      apiKey: user.apiKey,
+      limits,
+      usage: await this.usage(user.id),
+    }
+  }
+}

app/services/batch_store.ts

@@ -0,0 +1,314 @@
+import db from '@adonisjs/lucid/services/db'
+import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
+import { UPLOAD_GAP_MS, type EdgeBatch, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
+import ChunkPartitions from '#services/chunk_partitions'
+import { decodeChunk } from '#lib/chunks/index'
+
+export class StaleBatch extends Error {}
+
+export interface StoredBatch {
+  releases: number
+  newReleases: number
+  candidates: number
+  fragments: number
+}
+
+/**
+ * Stores one batch of a leased range (REBUILD.md, parts 3–7), in one
+ * transaction: release aggregates in the catalog, one chunk per release,
+ * probe candidates, rule counts and samples, edge fragments, and the
+ * range's resume point. A batch that doesn't start at the resume point was
+ * sent twice or too late, and is refused (StaleBatch), so the sums stay right.
+ */
+class BatchStore {
+  async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
+    await ChunkPartitions.ensure()
+    return db.transaction(async (trx) => {
+      // One batch of a range at a time, without holding the range's row for the whole transaction: the
+      // prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
+      // LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
+      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [rangeId])
+      const range = await trx.from('ranges').where('id', rangeId).first()
+      if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
+        throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
+      if (Number(range.next_id) !== batch.from)
+        throw new StaleBatch(
+          `Range ${rangeId} is at ${range.next_id}, the batch starts at ${batch.from}`
+        )
+      const result: StoredBatch = {
+        releases: batch.releases.length,
+        newReleases: 0,
+        candidates: 0,
+        fragments: batch.fragments.length,
+      }
+
+      const written = await this.writeReleases(
+        trx,
+        rangeId,
+        batch.from,
+        batch.releases,
+        batch.candidates
+      )
+      result.newReleases = written.newReleases
+      result.candidates = written.candidates
+
+      if (batch.counts.length > 0) {
+        await trx.rawQuery(
+          `INSERT INTO rule_counts (range_id, rule, key, articles, files)
+           SELECT ?, v.rule, v.key, v.articles, v.files FROM jsonb_to_recordset(?::jsonb) AS v(rule text, key text, articles bigint, files bigint)
+           ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
+          [rangeId, JSON.stringify(batch.counts)]
+        )
+      }
+
+      for (const s of batch.samples) {
+        await trx.insertQuery().table('rule_samples').insert({
+          range_id: rangeId,
+          rule: s.rule,
+          rule_key: s.ruleKey,
+          release_key: s.key,
+          name: s.name,
+          files: s.files,
+          segments: s.segments,
+          payload: s.payload,
+        })
+      }
+
+      for (let i = 0; i < batch.fragments.length; i += 2000) {
+        const rows = batch.fragments.slice(i, i + 2000).map((f) => ({
+          range_id: rangeId,
+          file_key: f.fileKey,
+          // Not Math.min(...): a big file's fragment has more segments than the call stack takes arguments.
+          article_number: f.segments.reduce((min, s) => (s[3] < min ? s[3] : min), Infinity),
+          segments: f.segments.length,
+          fragment: JSON.stringify(f),
+        }))
+        await trx.insertQuery().table('fragments').multiInsert(rows)
+      }
+
+      const moved = await trx
+        .from('ranges')
+        .where('id', rangeId)
+        .where('status', 'leased')
+        .where('node_id', nodeId)
+        .where('next_id', batch.from)
+        .update({
+          next_id: batch.next,
+          headers: db.raw('headers + ?', [batch.headers]),
+          // The range's posting time span (release_final): the batches' trimmed spans, min and max (least and greatest skip nulls).
+          posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [batch.postedLow ?? null]),
+          posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [batch.postedHigh ?? null]),
+          heartbeat_at: new Date(),
+        })
+      if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
+      return result
+    })
+  }
+
+  /**
+   * An upload spread over groups (FTD: par2 and part1 to a.b.music.misc,
+   * part2 to a.b.sounds.music, each article in one group only; LOG.md
+   * 2026-10-11 (9)): a chunk without a release in its own group joins one of
+   * its key in another group whose window is within UPLOAD_GAP_MS, when none
+   * of its file names are in that release yet. A post of the same files to
+   * another group (other message-ids, the same names) stays a release of its
+   * own, or the NZB would hold every file twice. Reads the candidate's file
+   * names from its chunks; candidates are few.
+   */
+  private async joinAcrossGroups(
+    trx: TransactionClientContract,
+    part: ReleaseChunk[],
+    windows: { idx: number; key: string; first: number | null | undefined; last: number | null | undefined }[],
+    groupId: number,
+    joined: Map<number, number>
+  ) {
+    const open = windows.filter((w) => !joined.has(w.idx) && w.first)
+    if (open.length === 0) return
+    const { rows: candidates } = await trx.rawQuery(
+      `SELECT v.idx, r.id
+       FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
+       JOIN releases r ON r.identity_key = v.key AND r.group_id <> ?
+       WHERE r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0)
+       ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
+      [JSON.stringify(open), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
+    )
+    if (candidates.length === 0) return
+    const ids: number[] = [...new Set<number>(candidates.map((c: { id: string }) => Number(c.id)))]
+    const names = new Map<number, Set<string>>()
+    const { rows: chunks } = await trx.rawQuery(`SELECT release_id, payload FROM release_chunks WHERE release_id = ANY(?)`, [ids])
+    for (const c of chunks) {
+      const set = names.get(Number(c.release_id)) ?? new Set<string>()
+      for (const f of decodeChunk(c.payload)) set.add(f.name)
+      names.set(Number(c.release_id), set)
+    }
+    for (const c of candidates as { idx: number; id: string }[]) {
+      if (joined.has(c.idx)) continue
+      const have = names.get(Number(c.id)) ?? new Set<string>()
+      const incoming = decodeChunk(part[c.idx].payload).map((f) => f.name)
+      if (incoming.every((name) => !have.has(name))) {
+        joined.set(c.idx, Number(c.id))
+        for (const name of incoming) have.add(name)
+        names.set(Number(c.id), have)
+      }
+    }
+  }
+
+  /**
+   * Release aggregates in the catalog, one write-once chunk per release, and
+   * probe candidates (a batch's, or files the fragment merger completed).
+   * A chunk joins the release of its key whose articles the range's group
+   * owns and whose posting window is within UPLOAD_GAP_MS of its own, else
+   * starts one: re-uploads are releases of their own (decision d). The
+   * group is locked, so concurrent writers don't both start one.
+   */
+  async writeReleases(
+    trx: TransactionClientContract,
+    rangeId: number,
+    batchFrom: number,
+    releases: ReleaseChunk[],
+    candidates: ProbeCandidate[]
+  ) {
+    const out = { newReleases: 0, candidates: 0 }
+    const ids = new Map<string, number>()
+    const slot = (key: string, upload: number) => `${key}\n${upload}`
+    const { group_id: groupId } = await trx.from('ranges').where('id', rangeId).select('group_id').firstOrFail()
+    // One writer per group at a time: a chunk only matches releases of its group, and a lock per key
+    // ran out of shared memory (a picture group's batch has 50k keys; LOG.md 2026-10-09 (12)).
+    await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('releases:' || ?::text, 0))`, [groupId])
+    const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
+    for (let i = 0; i < sorted.length; i += 1000) {
+      const part = sorted.slice(i, i + 1000)
+      const windows = part.map((r, idx) => ({ idx, key: r.key, first: r.postedAt, last: r.postedLast ?? r.postedAt }))
+      const { rows: matches } = await trx.rawQuery(
+        `SELECT DISTINCT ON (v.idx) v.idx, r.id
+         FROM jsonb_to_recordset(?::jsonb) AS v(idx int, key text, first bigint, last bigint)
+         JOIN releases r ON r.identity_key = v.key AND r.group_id = ?
+         WHERE v.first IS NULL OR r.posted_at IS NULL
+            OR (r.posted_at <= to_timestamp((v.last + ?) / 1000.0) AND coalesce(r.last_posted_at, r.posted_at) >= to_timestamp((v.first - ?) / 1000.0))
+         ORDER BY v.idx, r.last_posted_at DESC NULLS LAST, r.id DESC`,
+        [JSON.stringify(windows), groupId, UPLOAD_GAP_MS, UPLOAD_GAP_MS]
+      )
+      const joined = new Map<number, number>(matches.map((m: { idx: number; id: string }) => [m.idx, Number(m.id)]))
+      await this.joinAcrossGroups(trx, part, windows, groupId, joined)
+      const row = (r: ReleaseChunk) => ({
+        name: r.name,
+        source: r.source,
+        poster: r.poster,
+        groups: r.groups,
+        first: r.postedAt,
+        last: r.postedLast ?? r.postedAt,
+        file_total: r.fileTotal,
+        files: r.files,
+        segments: r.segments,
+        bytes: r.bytes,
+        first_article: r.firstArticle,
+        last_article: r.lastArticle,
+        readable: r.readable,
+        has_par2: r.hasPar2,
+      })
+      const updates = part.flatMap((r, idx) => (joined.has(idx) ? [{ id: joined.get(idx)!, ...row(r) }] : []))
+      if (updates.length > 0) {
+        await trx.rawQuery(
+          `WITH v AS (SELECT * FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)),
+                a AS (SELECT id, sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, count(*) AS chunks, min(first) AS first, max(last) AS last,
+                             max(file_total) AS file_total, min(first_article) AS first_article, max(last_article) AS last_article, bool_or(readable) AS readable, bool_or(has_par2) AS has_par2
+                      FROM v GROUP BY id)
+           UPDATE releases r SET
+             files = r.files + a.files,
+             segments = r.segments + a.segments,
+             bytes = r.bytes + a.bytes,
+             chunks = r.chunks + a.chunks,
+             posted_at = least(r.posted_at, to_timestamp(a.first / 1000.0)),
+             last_posted_at = greatest(r.last_posted_at, to_timestamp(a.last / 1000.0)),
+             file_total = coalesce(r.file_total, a.file_total),
+             first_article = least(r.first_article, a.first_article),
+             last_article = greatest(r.last_article, a.last_article),
+             readable = r.readable OR a.readable,
+             has_par2 = r.has_par2 OR a.has_par2,
+             -- As it stands (the counter's files all found); settling makes it final.
+             complete = CASE WHEN coalesce(r.file_total, a.file_total) IS NULL THEN NULL ELSE r.files + a.files >= coalesce(r.file_total, a.file_total) END,
+             last_seen_at = now()
+           FROM a WHERE r.id = a.id`,
+          [JSON.stringify(updates)]
+        )
+        part.forEach((r, idx) => joined.has(idx) && ids.set(slot(r.key, r.upload), joined.get(idx)!))
+      }
+      const fresh = part.filter((_, idx) => !joined.has(idx))
+      if (fresh.length > 0) {
+        // The first upload of a key keeps the key; later ones are key:2, key:3, …
+        const { rows: counts } = await trx.rawQuery(`SELECT identity_key, count(*) AS n FROM releases WHERE identity_key = ANY(?) GROUP BY 1`, [[...new Set(fresh.map((r) => r.key))]])
+        const taken = new Map<string, number>(counts.map((c: { identity_key: string; n: string }) => [c.identity_key, Number(c.n)]))
+        const numbered = (r: ReleaseChunk) => {
+          const n = (taken.get(r.key) ?? 0) + 1
+          taken.set(r.key, n)
+          return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
+        }
+        // Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
+        const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
+        if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
+        let inserts = fresh.map(numbered)
+        // Writers of other groups number the same key at the same time (a spam post crossposted to 67
+        // groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
+        for (let attempt = 0; inserts.length > 0; attempt++) {
+          if (attempt === 20) throw new Error(`No free release key for ${inserts[0].key}`)
+          const { rows } = await trx.rawQuery(
+            `INSERT INTO releases (release_key, identity_key, group_id, name, source, poster, groups, posted_at, last_posted_at, file_total, files, segments, bytes, chunks, first_article, last_article, readable, has_par2, complete)
+             SELECT v.release_key, v.key, ?, v.name, v.source, v.poster, v.groups, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
+                    v.first_article, v.last_article, v.readable, v.has_par2, CASE WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END
+             FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, key text, name text, source text, poster text, groups jsonb, first bigint, last bigint, file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean)
+             ON CONFLICT (release_key) DO NOTHING
+             RETURNING id, release_key`,
+            [groupId, JSON.stringify(inserts)]
+          )
+          const byReleaseKey = new Map<string, number>(rows.map((x: { id: string; release_key: string }) => [x.release_key, Number(x.id)]))
+          const missed: typeof inserts = []
+          for (const r of inserts) {
+            const id = byReleaseKey.get(r.release_key)
+            if (id === undefined) missed.push(r)
+            else ids.set(slot(r.key, r.upload), id)
+          }
+          out.newReleases += rows.length
+          const chunks = new Map(fresh.map((r) => [slot(r.key, r.upload), r]))
+          inserts = missed.map((r) => numbered(chunks.get(slot(r.key, r.upload))!))
+        }
+      }
+    }
+
+    // One write-once chunk per release.
+    for (let i = 0; i < sorted.length; i += 500) {
+      const rows = sorted.slice(i, i + 500).map((r) => ({
+        release_id: ids.get(slot(r.key, r.upload))!,
+        range_id: rangeId,
+        batch_from: batchFrom,
+        files: r.files,
+        segments: r.segments,
+        payload: r.payload,
+      }))
+      if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
+    }
+
+    const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
+    for (let i = 0; i < kept.length; i += 2000) {
+      const rows = kept
+        .slice(i, i + 2000)
+        .map((c) => ({
+          release_id: ids.get(slot(c.releaseKey, c.upload ?? 0))!,
+          kind: c.kind,
+          message_id: c.messageId,
+          file: c.file,
+        }))
+      const added = await trx
+        .knexQuery()
+        .table('probe_candidates')
+        .insert(rows)
+        .onConflict(['release_id', 'kind', 'message_id'])
+        .ignore()
+        .returning('id')
+      out.candidates += added.length
+    }
+
+    return out
+  }
+}
+
+export default new BatchStore()

app/services/catalog_space.ts

@@ -0,0 +1,38 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+
+/** How often the catalog's size is measured (pg_tablespace_size walks its files). */
+const CHECK_MS = 60_000
+
+/**
+ * The catalog's room on the NVMe (CATALOG_MAX_NVME_GB, like voyager 1's
+ * BACKFILL_MAX_NVME_GB): once its tablespace (the database's
+ * default_tablespace) is over the cap, only the tail and the stored ranges
+ * (priority below 0) are leased and prefetched, and history waits. After
+ * reset 24 the catalog grew ~3.7 GB an hour, root down to 31 GB free with
+ * 600k history ranges to go (LOG.md 2026-10-14 (53)). Off when unset.
+ */
+class CatalogSpace {
+  private checkedAt = 0
+  private over = false
+
+  async full(): Promise<boolean> {
+    const cap = coordinatorConfig.catalogMaxNvmeGb
+    if (!cap) return false
+    if (Date.now() - this.checkedAt < CHECK_MS) return this.over
+    this.checkedAt = Date.now()
+    const { rows } = await db.rawQuery(
+      `SELECT pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
+    )
+    const bytes = Number(rows[0].bytes)
+    const was = this.over
+    this.over = bytes > cap * 1e9
+    if (this.over !== was) {
+      logger.info(`Catalog ${(bytes / 1e9).toFixed(1)} GB of ${cap}: history ${this.over ? 'paused (tail and stored ranges only)' : 'resumed'}.`)
+    }
+    return this.over
+  }
+}
+
+export default new CatalogSpace()

app/services/chunk_partitions.ts

@@ -0,0 +1,30 @@
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+
+/** Months ahead that always have a release_chunks partition. */
+const AHEAD = 2
+
+/**
+ * release_chunks is partitioned by month of writing (REBUILD.md, decision a).
+ * Inserts fail without a partition for now(), so the current and the next
+ * months' partitions are created before they are needed.
+ */
+class ChunkPartitions {
+  async ensure(now = new Date()) {
+    // Checked each time, not remembered: a rolled-back transaction can take a created partition along.
+    const current = `release_chunks_${now.toISOString().slice(0, 7).replace('-', '')}`
+    const { rows } = await db.rawQuery(`SELECT to_regclass(?) IS NOT NULL AS present`, [current])
+    if (rows[0].present) return
+    const tablespace = env.get('CHUNK_TABLESPACE')
+    for (let i = 0; i <= AHEAD; i++) {
+      const from = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + i, 1))
+      const to = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + i + 1, 1))
+      const name = `release_chunks_${from.toISOString().slice(0, 7).replace('-', '')}`
+      await db.rawQuery(
+        `CREATE TABLE IF NOT EXISTS ${name} PARTITION OF release_chunks FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')${tablespace ? ` TABLESPACE ${tablespace}` : ''}`
+      )
+    }
+  }
+}
+
+export default new ChunkPartitions()

app/services/corpus.ts

@@ -0,0 +1,90 @@
+import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { unzip } from 'node:zlib'
+import { promisify } from 'node:util'
+import coordinatorConfig from '#config/coordinator'
+import type { ChunkSpan } from '#services/spool'
+
+const inflate = promisify(unzip)
+
+export interface CorpusRange {
+  group: string
+  start: number
+  end: number
+}
+
+/**
+ * Stored XOVER chunks of the ranges marked `capture` (REBUILD.md, corpus):
+ * `<dir>/<group>/<start>-<end>/<nnnn>-<from>-<to>.gz` with a range.json once
+ * every chunk is there, the layout edge-bench reads (edge/fixtures). The
+ * grouping sweeps replay them with each build's rules instead of waiting for
+ * a re-index; the prefetcher reads a stored chunk instead of fetching it
+ * again. Off without CORPUS_DIR.
+ */
+class Corpus {
+  get dir(): string | undefined {
+    return coordinatorConfig.corpus.dir
+  }
+
+  private rangeDir(range: CorpusRange) {
+    return join(this.dir!, range.group, `${range.start}-${range.end}`)
+  }
+
+  private path(range: CorpusRange, span: ChunkSpan) {
+    const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
+    return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
+  }
+
+  /** Whether every chunk of the range is stored (its range.json). */
+  async complete(range: CorpusRange): Promise<boolean> {
+    if (!this.dir) return false
+    return access(join(this.rangeDir(range), 'range.json')).then(
+      () => true,
+      () => false
+    )
+  }
+
+  /** Whether one chunk is stored. */
+  async has(range: CorpusRange, span: ChunkSpan): Promise<boolean> {
+    if (!this.dir) return false
+    return access(this.path(range, span)).then(
+      () => true,
+      () => false
+    )
+  }
+
+  /** A stored chunk, or null. */
+  async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
+    if (!this.dir) return null
+    return readFile(this.path(range, span)).catch(() => null)
+  }
+
+  /**
+   * Stores a chunk that inflates (a reply cut short isn't kept), and the
+   * range.json once the range is complete (`total` chunks).
+   */
+  async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
+    if (!this.dir) return
+    try {
+      await inflate(data)
+    } catch {
+      return
+    }
+    const dir = this.rangeDir(range)
+    const file = this.path(range, span)
+    await mkdir(dir, { recursive: true })
+    await writeFile(`${file}.part`, data)
+    await rename(`${file}.part`, file)
+    const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
+    if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
+  }
+
+  /** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
+  async removeChunk(range: CorpusRange, span: ChunkSpan) {
+    if (!this.dir) return
+    await rm(this.path(range, span), { force: true })
+    await rm(join(this.rangeDir(range), 'range.json'), { force: true })
+  }
+}
+
+export default new Corpus()

app/services/fragment_merger.ts

@@ -0,0 +1,153 @@
+import db from '@adonisjs/lucid/services/db'
+import BatchStore from '#services/batch_store'
+import ChunkPartitions from '#services/chunk_partitions'
+import RuleSets from '#services/rule_sets'
+import { RuleSet } from '#lib/rules/index'
+import { candidatesOf, chunkOf, mergeFragments, releaseKeyOf, uploadsOf, type Fragment, type ProbeCandidate, type ReleaseChunk } from '#lib/edge/index'
+import type { AggregatedFile } from '#lib/files/index'
+import type { EdgeSegment } from '#lib/edge/index'
+
+export interface MergeResult {
+  files: number
+  completed: number
+  incomplete: number
+  waiting: number
+  /** This round reached the end of the file keys (the next one starts over). */
+  wrapped: boolean
+}
+
+/** File keys per round. */
+const BATCH = 20_000
+
+/**
+ * Files cut by a range's edge (REBUILD.md, fragments), put together once
+ * the ranges around them are done: a complete file is judged by the rules
+ * like any other and written as a release chunk (with probe candidates); a
+ * file still missing parts while every neighbouring range is done is
+ * incomplete, and dropped (counted on its ranges). Others wait.
+ */
+class FragmentMerger {
+  /** Where the next round goes on: waiting files must not take every round's places. */
+  private cursor = ''
+
+  async run(): Promise<MergeResult> {
+    const result: MergeResult = { files: 0, completed: 0, incomplete: 0, waiting: 0, wrapped: false }
+    const { rows: keys } = await db.rawQuery(
+      `SELECT DISTINCT f.file_key FROM fragments f JOIN ranges r ON r.id = f.range_id
+       WHERE r.status = 'done' AND f.file_key > ? ORDER BY f.file_key LIMIT ?`,
+      [this.cursor, BATCH]
+    )
+    // At the end of the keys: the next round starts over.
+    result.wrapped = keys.length < BATCH
+    this.cursor = result.wrapped ? '' : keys.at(-1).file_key
+    if (keys.length === 0) return result
+    const rules = new RuleSet(await RuleSets.current())
+    const rows = await db
+      .from('fragments as f')
+      .join('ranges as r', 'r.id', 'f.range_id')
+      .whereIn('f.file_key', keys.map((k: { file_key: string }) => k.file_key))
+      .select('f.id', 'f.file_key', 'f.range_id', 'f.fragment', 'r.group_id', 'r.start_id', 'r.end_id', 'r.status')
+    const byKey = new Map<string, typeof rows>()
+    for (const row of rows) {
+      const list = byKey.get(row.file_key)
+      if (list) list.push(row)
+      else byKey.set(row.file_key, [row])
+    }
+
+    const completed: { file: AggregatedFile<EdgeSegment>; rangeId: number; ids: number[] }[] = []
+    const incomplete: { rangeId: number; ids: number[]; segments: number }[] = []
+    const neighbours = await this.neighbourStatus([...byKey.values()])
+    for (const list of byKey.values()) {
+      result.files++
+      const file = mergeFragments(list.map((r) => r.fragment as Fragment))
+      const ids = list.map((r) => Number(r.id))
+      const rangeId = Math.max(...list.map((r) => Number(r.range_id)))
+      if (file) {
+        completed.push({ file, rangeId, ids })
+        continue
+      }
+      if (list.every((r) => r.status === 'done') && neighbours(list)) {
+        incomplete.push({ rangeId, ids, segments: list.reduce((n, r) => n + (r.fragment as Fragment).segments.length, 0) })
+      } else {
+        result.waiting++
+      }
+    }
+
+    // Completed files, grouped into their releases, judged and written like a batch's.
+    const byRelease = new Map<string, { files: AggregatedFile<EdgeSegment>[]; rangeId: number }>()
+    for (const { file, rangeId } of completed) {
+      const key = releaseKeyOf(file.segments[0])
+      const entry = byRelease.get(key)
+      if (entry) entry.files.push(file)
+      else byRelease.set(key, { files: [file], rangeId })
+    }
+    const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
+    for (const [key, { files, rangeId }] of byRelease) {
+      const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
+      if (verdict.rule !== null) continue
+      uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
+    }
+    await ChunkPartitions.ensure()
+    await db.transaction(async (trx) => {
+      // In key order (batches lock their group; the merger only ever waits on one batch at a time).
+      releases.sort((a, b) => a.chunk.key.localeCompare(b.chunk.key) || a.chunk.upload - b.chunk.upload)
+      for (const { rangeId, chunk, candidates } of releases) await BatchStore.writeReleases(trx, rangeId, 0, [chunk], candidates)
+      const done = [...completed.flatMap((c) => c.ids), ...incomplete.flatMap((c) => c.ids)]
+      for (let i = 0; i < done.length; i += 5000) await trx.from('fragments').whereIn('id', done.slice(i, i + 5000)).delete()
+      for (const { rangeId, segments } of incomplete) {
+        await trx.rawQuery(
+          `INSERT INTO rule_counts (range_id, rule, key, articles, files) VALUES (?, 'incomplete', 'edge', ?, 1)
+           ON CONFLICT (range_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + 1`,
+          [rangeId, segments]
+        )
+      }
+    })
+    result.completed = completed.length
+    result.incomplete = incomplete.length
+    return result
+  }
+
+  /**
+   * For each file's fragments: whether the ranges right before and after
+   * their ranges (where a missing part could be) are done, read in one
+   * query for the whole round (per file it was three, ~2,000 files a
+   * minute). A neighbour not planned yet counts as not done (newer articles
+   * come later), except before the group's oldest article.
+   */
+  private async neighbourStatus(lists: { group_id: number; start_id: number | string; end_id: number | string }[][]) {
+    const span = (list: (typeof lists)[number]) => ({
+      group: Number(list[0].group_id),
+      first: Math.min(...list.map((r) => Number(r.start_id))),
+      last: Math.max(...list.map((r) => Number(r.end_id))),
+    })
+    const spans = lists.map(span)
+    const groups = [...new Set(spans.map((s) => s.group))]
+    // The range holding the article before and the one after, each by the (group_id, start_id) index:
+    // "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09).
+    const { rows } = await db.rawQuery(
+      `SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after
+       FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start)
+       LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end
+                          ORDER BY r.start_id DESC LIMIT 1) b ON true
+       LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start
+                          ORDER BY r.start_id DESC LIMIT 1) a ON true`,
+      [spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)]
+    )
+    const byEnd = new Map<string, string>()
+    const byStart = new Map<string, string>()
+    for (const r of rows) {
+      if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before)
+      if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after)
+    }
+    const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)]))
+    return (list: (typeof lists)[number]) => {
+      const { group, first, last } = span(list)
+      const before = byEnd.get(`${group}:${first - 1}`)
+      const low = lows.get(group)
+      const beforeDone = before !== undefined ? before === 'done' : low === null || low === undefined || first <= low
+      return beforeDone && byStart.get(`${group}:${last + 1}`) === 'done'
+    }
+  }
+}
+
+export default new FragmentMerger()

app/services/leases.ts

@@ -0,0 +1,109 @@
+import db from '@adonisjs/lucid/services/db'
+import coordinatorConfig from '#config/coordinator'
+import RuleSets from '#services/rule_sets'
+import Spool from '#services/spool'
+import Corpus from '#services/corpus'
+import CatalogSpace from '#services/catalog_space'
+import type { Lease } from '#lib/edge/index'
+
+export class LeaseLost extends Error {}
+
+/**
+ * Ranges for workers (REBUILD.md, coordinator): the next range with chunks
+ * spooled, by priority (oldest era first), else the next planned one (the
+ * prefetcher fetches leased ranges first; the worker's stream waits). A
+ * worker never waits for prefetching to choose its range: refused leases
+ * left workers idle for 30 s, so fewer ranges were leased and fewer
+ * prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
+ * LEASE_TTL_SECONDS is taken over and resumes at its resume point.
+ */
+/**
+ * A group's batches wait for each other (the batch store locks the group): more workers than this on one group only
+ * queue. Eight on one picture group's gap ranges (priority -1, in id order) held the node API's batches for minutes
+ * (LOG.md 2026-10-16 (68)).
+ */
+const MAX_LEASES_PER_GROUP = 2
+
+class Leases {
+  async lease(nodeId: number): Promise<Lease | null> {
+    const rules = await RuleSets.current()
+    // The catalog at its cap on the NVMe: the tail and the stored ranges only (CatalogSpace).
+    const history = (await CatalogSpace.full()) ? 'AND priority < 0' : ''
+    const take = (spooled: boolean) =>
+      db.rawQuery(
+        `UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
+         WHERE r.id = (
+           SELECT id FROM ranges
+           WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id ${history}
+             AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
+             AND (SELECT count(*) FROM ranges l
+                  WHERE l.status = 'leased' AND l.group_id = ranges.group_id AND l.id <> ranges.id
+                    AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
+           ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
+         RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
+        [nodeId, rules.version, coordinatorConfig.leaseTtlSeconds, coordinatorConfig.leaseTtlSeconds]
+      )
+    let { rows } = await take(true)
+    if (rows.length === 0) ({ rows } = await take(false))
+    const row = rows[0]
+    if (!row) return null
+    await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
+    const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
+    return { range: { id: Number(row.id), group: group.name, start: Number(row.start_id), end: Number(row.end_id), next: Number(row.next_id) }, rules }
+  }
+
+  async leased(rangeId: number, nodeId: number) {
+    const range = await db.from('ranges').where('id', rangeId).first()
+    if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
+    return range
+  }
+
+  async heartbeat(rangeId: number, nodeId: number) {
+    await this.leased(rangeId, nodeId)
+    await db.from('ranges').where('id', rangeId).update({ heartbeat_at: new Date() })
+    await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
+  }
+
+  /** The worker reached the range's end: done, and its spooled chunks go. */
+  async finish(rangeId: number, nodeId: number) {
+    const range = await this.leased(rangeId, nodeId)
+    if (Number(range.next_id) <= Number(range.end_id)) throw new LeaseLost(`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`)
+    await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
+    await Spool.remove(rangeId)
+  }
+
+  /**
+   * A worker got a chunk that doesn't inflate: the spooled chunk goes and
+   * the range back to the queue (the prefetcher fetches leased ranges'
+   * missing chunks again; the next worker resumes at the range's resume point).
+   */
+  async corrupt(rangeId: number, nodeId: number, from: number) {
+    const range = await this.leased(rangeId, nodeId)
+    const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find((s) => s.from === from)
+    if (span) {
+      await Spool.removeChunk(rangeId, span)
+      if (range.capture) {
+        const group = await db.from('groups').where('id', range.group_id).select('name').first()
+        await Corpus.removeChunk({ group: group.name, start: Number(range.start_id), end: Number(range.end_id) }, span)
+      }
+    }
+    const plain: number[] = (range.plain_chunks ?? []).map(Number)
+    // Reported once already: the compressed copy may have been in flight (or the prefetcher's targets not
+    // refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
+    if (plain.filter((p) => p === from).length >= 2) {
+      // Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
+      await db.from('ranges').where('id', rangeId).update({ status: 'failed', node_id: null, error: `chunk ${from} doesn't inflate, also fetched plain` })
+      return
+    }
+    // The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
+    await db.rawQuery(`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`, [from, rangeId])
+  }
+
+  /** A stopping worker gives its range back; the next one resumes it. */
+  async release(rangeId: number, nodeId: number) {
+    await this.leased(rangeId, nodeId)
+    await db.from('ranges').where('id', rangeId).update({ status: 'planned', node_id: null })
+  }
+}
+
+export default new Leases()

app/services/nntp_pool.ts

@@ -0,0 +1,150 @@
+import env from '#start/env'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+
+/**
+ * The coordinator's header connections (REBUILD.md, decision e: workers
+ * have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
+ * that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
+ * replaced on the next use.
+ */
+class NntpPool {
+  private idle: XoverClient[] = []
+  private busy = new Set<XoverClient>()
+  private open = 0
+  private waiting: ((client: XoverClient) => void)[] = []
+  private openedAt = new Map<XoverClient, number>()
+  /** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
+  private retireAt = new Map<XoverClient, number>()
+
+  get size() {
+    return coordinatorConfig.nntpConnections
+  }
+
+  async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
+    return this.run(await this.acquire(), work)
+  }
+
+  /** As use(), if a connection is free or can be opened now; else null, without waiting. */
+  tryUse<T>(work: (client: XoverClient) => Promise<T>): Promise<T> | null {
+    if (this.idle.length === 0 && this.open >= this.size) return null
+    return this.acquire().then((client) => this.run(client, work))
+  }
+
+  /** Connections with a command in flight (the prefetcher's, the planner's). */
+  get busyCount() {
+    return this.busy.size
+  }
+
+  /** How long ago the connection was opened, in ms. */
+  ageOf(client: XoverClient): number {
+    return Date.now() - (this.openedAt.get(client) ?? Date.now())
+  }
+
+  private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
+    this.busy.add(client)
+    let broken = false
+    try {
+      return await work(client)
+    } catch (error) {
+      broken = true
+      throw error
+    } finally {
+      this.busy.delete(client)
+      this.release(client, broken)
+    }
+  }
+
+  private async acquire(): Promise<XoverClient> {
+    const ready = this.idle.pop()
+    if (ready) return ready
+    if (this.open < this.size) {
+      this.open++
+      try {
+        const client = await XoverClient.connect({
+          host: env.get('NNTP_HOST')!,
+          port: env.get('NNTP_PORT', 563),
+          user: env.get('NNTP_USER'),
+          password: env.get('NNTP_PASSWORD'),
+          secure: env.get('NNTP_SECURE', true),
+          compress: true,
+          timeoutMs: coordinatorConfig.nntpTimeoutMs,
+        })
+        this.openedAt.set(client, Date.now())
+        const maxMs = coordinatorConfig.nntpConnectionMaxMs
+        if (maxMs > 0) this.retireAt.set(client, Date.now() + maxMs * (0.75 + Math.random() / 2))
+        return client
+      } catch (error) {
+        this.open--
+        throw error
+      }
+    }
+    return new Promise((resolve) => this.waiting.push(resolve))
+  }
+
+  private release(client: XoverClient, broken: boolean) {
+    if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
+      this.open--
+      this.openedAt.delete(client)
+      this.retireAt.delete(client)
+      try {
+        client.close()
+      } catch (error: any) {
+        logger.debug(`Closing a failed NNTP connection: ${error.message}`)
+      }
+      // Someone waiting gets a fresh connection.
+      const next = this.waiting.shift()
+      if (next) this.acquire().then(next, () => this.waiting.unshift(next))
+      return
+    }
+    const next = this.waiting.shift()
+    if (next) next(client)
+    else this.idle.push(client)
+  }
+
+  /** Closes every connection, also those in use: their commands fail. */
+  private plain: Promise<XoverClient> | null = null
+  private plainQueue: Promise<unknown> = Promise.resolve()
+
+  /**
+   * One connection without XFEATURE COMPRESS GZIP, opened when first needed,
+   * one command at a time: for chunks whose compressed reply the server
+   * cuts short (Leases.corrupt). Its replies are deflated here.
+   */
+  async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
+    const run = this.plainQueue.then(async () => {
+      this.plain ??= XoverClient.connect({
+        host: env.get('NNTP_HOST')!,
+        port: env.get('NNTP_PORT', 563),
+        user: env.get('NNTP_USER'),
+        password: env.get('NNTP_PASSWORD'),
+        secure: env.get('NNTP_SECURE', true),
+        compress: false,
+        timeoutMs: coordinatorConfig.nntpTimeoutMs,
+      })
+      const client = await this.plain
+      try {
+        return await work(client)
+      } catch (error) {
+        this.plain = null
+        client.close()
+        throw error
+      }
+    })
+    this.plainQueue = run.catch(() => {})
+    return run
+  }
+
+  async close() {
+    void this.plain?.then((c) => c.close()).catch(() => {})
+    this.plain = null
+    for (const client of [...this.idle.splice(0), ...this.busy]) client.close()
+    this.busy.clear()
+    this.openedAt.clear()
+    this.retireAt.clear()
+    this.open = 0
+  }
+}
+
+export default new NntpPool()

app/services/nzb_service.ts

@@ -0,0 +1,44 @@
+import { gunzipSync } from 'node:zlib'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
+import { buildNzb } from '#lib/nzb/index'
+
+/**
+ * A release's files and NZB (REBUILD.md, part 9), from its chunks in the
+ * chunk store: every chunk decoded, its files in name order. A file stored
+ * twice (the same name and first article, e.g. a range read again) is taken
+ * once.
+ */
+class NzbService {
+  async files(releaseId: number): Promise<ChunkFile[]> {
+    const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
+    const files = new Map<string, ChunkFile>()
+    for (const chunk of chunks) {
+      for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
+    }
+    return [...files.values()].sort((a, b) => a.name.localeCompare(b.name))
+  }
+
+  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
+    const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
+    if (!release) return null
+    // A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
+    if (release.source === 'spot') {
+      const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
+      if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') }
+    }
+    const xml = buildNzb(
+      (await this.files(releaseId)).map((f) => ({
+        name: f.name,
+        poster: f.poster,
+        date: f.postedAt ?? 0,
+        groups: f.groups,
+        segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
+      })),
+      { title: release.name }
+    )
+    return { name: release.name, xml }
+  }
+}
+
+export default new NzbService()

app/services/par2_merger.ts

@@ -0,0 +1,315 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { par2ReleaseName } from '#lib/release/index'
+import { looksRandom } from '#lib/release/random'
+import { SidecarFolder } from '#services/sidecar_folder'
+import { Par2Pruner } from '#services/par2_pruner'
+
+/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+/**
+ * A set held by more releases nearby than this is one posted again and again, not one upload's par2 (every
+ * "kantodoNNN.bin" test upload carries the same set: 64 releases became one of 631 files; LOG.md 2026-10-12 (30)).
+ */
+const MAX_HOLDERS = 4
+/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
+const MAX_FILES_PER_SET_FILE = 3
+const SLACK_FILES = 10
+/** Between rounds. */
+const ROUND_MS = 60_000
+/** A round looks back this far past the previous one's start (rows committed while it ran). */
+const OVERLAP_MS = 60_000
+/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
+const FOLD_EVERY_MS = 5 * 60_000
+/** Between par2_files prunes (complete and judged sets' file lists). */
+const PRUNE_EVERY_MS = 60 * 60_000
+/** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
+const CURSOR = 'merger'
+
+export interface MergeRound {
+  sets: number
+  merges: number
+  releases: number
+  /** The inner-name pass: stems checked, merges, releases in them. */
+  stems: number
+  stemMerges: number
+  stemReleases: number
+  /** The sidecar fold: candidates looked at, sidecar-only ones among them, folded into their upload. */
+  folded?: { looked: number; sidecars: number; folded: number }
+}
+
+/**
+ * A lone inspection's inner stem: its yEnc name up to the first dot, when 16–64 letters and digits
+ * ("bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001"). The same expression as release_inspections_inner_stem_idx.
+ */
+const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
+
+interface Member {
+  id: number
+  release_key: string
+  group_id: number
+  files: number
+  posted_at: Date | null
+  readable: boolean
+  poster?: string | null
+  file_total?: number | null
+}
+
+/**
+ * The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
+ * describes are one upload. Obfuscated uploads post each file under a name of
+ * its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
+ * one-file releases), so the header pass can't group them; the prober reads
+ * each such file's first article (a `lone` candidate) and stores the MD5 of
+ * its first 16 KiB, and a par2 body's file list holds the same hash per file,
+ * whatever the file was renamed to. Releases in one group, posted within
+ * MAX_SPREAD_MS of a release that holds the set's par2, become the one with
+ * the most files: its chunks, probe candidates, inspections and hashes, the
+ * sums of its counts, and when none of them has a readable name the name the
+ * par2's file names share. Each merge is kept in release_merges.
+ *
+ * A second pass merges by inner name: the yEnc name in a lone file's first
+ * article keeps the upload's random stem when the subject names each file
+ * apart ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
+ * hep, hephaestus, gyges: every stem under one counter total; LOG.md
+ * 2026-10-12 (32)). Releases whose lone inspections share such a stem, in one
+ * group, by one poster, under the same counter total and within MAX_SPREAD_MS,
+ * are one upload. Only random stems: a readable one is grouped from the
+ * headers already, and a repost would share it.
+ */
+export class Par2Merger {
+  private stopped = false
+  private folder = new SidecarFolder((trx, found, key, via) => this.merge(trx, found, key, via))
+  /** Null until read from worker_cursors (1970 the first time: everything). */
+  private since: Date | null = null
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    let prunedAt = 0
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        // First, at start too: a prune needn't wait for a long round.
+        if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
+          prunedAt = Date.now()
+          const pruned = await new Par2Pruner().round()
+          if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
+        }
+        const round = await this.round()
+        if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
+        if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
+        if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
+      } catch (error: any) {
+        logger.error(`Par2 merge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
+  async round(): Promise<MergeRound> {
+    const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
+    if (!this.since) {
+      const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+      this.since = saved ? new Date(saved.at) : new Date(0)
+    }
+    const since = new Date(this.since.getTime() - OVERLAP_MS)
+    // First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
+    // par2 pass (hours when that starts over).
+    const folded = await this.folder.round()
+    let foldedAt = Date.now()
+    const foldAgain = async () => {
+      if (Date.now() - foldedAt < FOLD_EVERY_MS) return
+      const more = await this.folder.round()
+      folded.looked += more.looked
+      folded.sidecars += more.sidecars
+      folded.folded += more.folded
+      foldedAt = Date.now()
+      if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
+    }
+    const { rows } = await db.rawQuery(
+      `SELECT set_id FROM par2_sets WHERE created_at >= ?
+       UNION
+       SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
+      [since, since]
+    )
+    const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
+    for (const { set_id: setId } of rows) {
+      if (this.stopped) return round
+      await foldAgain()
+      // One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
+      const merged = await this.mergeSet(setId.trim()).catch((error) => {
+        logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
+        return 0
+      })
+      if (merged > 0) {
+        round.merges++
+        round.releases += merged + 1
+      }
+    }
+    const { rows: stems } = await db.rawQuery(
+      `SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i WHERE i.kind = 'lone' AND i.created_at >= ? AND ${INNER_STEM} IS NOT NULL`,
+      [since]
+    )
+    for (const { stem } of stems) {
+      if (this.stopped) return round
+      if (!looksRandom(stem)) continue
+      round.stems++
+      const merged = await this.mergeStem(stem).catch((error) => {
+        logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
+        return 0
+      })
+      if (merged > 0) {
+        round.stemMerges++
+        round.stemReleases += merged + 1
+      }
+    }
+    // Only after a whole round: a restart halfway goes on from the round before.
+    this.since = now[0].at
+    await db.rawQuery(
+      `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+      [CURSOR, this.since]
+    )
+    return round
+  }
+
+  /** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
+  async mergeStem(stem: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows } = await trx.rawQuery(
+        `SELECT DISTINCT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable, r.poster, r.file_total
+         FROM release_inspections i JOIN releases r ON r.id = i.release_id
+         WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
+        [stem]
+      )
+      const sorted: Member[] = rows.sort((a: Member, b: Member) => (a.posted_at ? new Date(a.posted_at).getTime() : 0) - (b.posted_at ? new Date(b.posted_at).getTime() : 0))
+      const anchor = sorted[0]
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
+      const near = (r: Member) =>
+        r.group_id === anchor.group_id &&
+        (r.poster ?? '') === (anchor.poster ?? '') &&
+        (total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
+        (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
+      const found = new Map<string, Member>()
+      for (const r of sorted.filter(near)) found.set(String(r.id), r)
+      if (found.size < 2) return 0
+      // No more files than the counter says, with room for par2 volumes posted under it.
+      const counted = Math.max(0, ...[...found.values()].map((r) => total(r) ?? 0))
+      const files = [...found.values()].reduce((n, m) => n + Number(m.files), 0)
+      if (counted > 0 && files > counted + SLACK_FILES) return 0
+      return this.merge(trx, found, stem, 'inner')
+    })
+  }
+
+  /** Merges the releases of one set; how many went into another. */
+  async mergeSet(setId: string): Promise<number> {
+    return db.transaction(async (trx) => {
+      const { rows: holders } = await trx.rawQuery(
+        `SELECT r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
+         WHERE s.set_id = ? ORDER BY r.id`,
+        [setId]
+      )
+      const anchor: Member | undefined = holders[0]
+      if (!anchor) return 0
+      const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
+      const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
+      const { rows: matches } = await trx.rawQuery(
+        `SELECT f.file_id, r.id, r.release_key, r.group_id, r.files, r.posted_at, r.readable
+         FROM par2_files f JOIN file_hashes h ON h.hash16k = f.hash16k AND h.size = f.length JOIN releases r ON r.id = h.release_id
+         WHERE f.set_id = ?`,
+        [setId]
+      )
+      // A file of the set that matches more than one release nearby is ambiguous: files that share their first
+      // 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became
+      // one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too.
+      const byFile = new Map<string, Map<string, Member>>()
+      for (const m of matches.filter(near)) {
+        const releases = byFile.get(m.file_id) ?? new Map<string, Member>()
+        releases.set(String(m.id), m)
+        byFile.set(m.file_id, releases)
+      }
+      const nearHolders = holders.filter(near)
+      if (nearHolders.length > MAX_HOLDERS) return 0
+      const found = new Map<string, Member>()
+      for (const r of nearHolders) found.set(String(r.id), r)
+      for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r)
+      if (found.size < 2) return 0
+      const { rows: described } = await trx.rawQuery(`SELECT count(*) AS n FROM par2_files WHERE set_id = ?`, [setId])
+      const setFiles = Math.max(Number(described[0].n), 1)
+      if ([...found.values()].reduce((n, m) => n + Number(m.files), 0) > MAX_FILES_PER_SET_FILE * setFiles + SLACK_FILES) return 0
+      return this.merge(trx, found, setId, 'par2')
+    })
+  }
+
+  /**
+   * Merges the found releases into the one with the most files (named by the par2 set's files when none
+   * is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
+   */
+  private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
+    {
+      // Locked in id order (the batch store and other rounds take the same rows).
+      const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
+      const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!)
+      if (members.length < 2) return 0
+      const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
+      const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
+      const name = via === 'par2' && members.every((m) => !m.readable) ? await this.setName(trx, key) : undefined
+
+      await trx.rawQuery(
+        `INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
+         SELECT id, release_key, ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
+        [target.id, key, via, others]
+      )
+      for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
+        await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      }
+      // A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
+      // round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
+      await trx.rawQuery(
+        `DELETE FROM par2_sets p WHERE p.release_id = ANY(?)
+           AND (EXISTS (SELECT 1 FROM par2_sets t WHERE t.set_id = p.set_id AND t.release_id = ?)
+             OR EXISTS (SELECT 1 FROM par2_sets o WHERE o.set_id = p.set_id AND o.release_id = ANY(?) AND o.release_id < p.release_id))`,
+        [others, target.id, others]
+      )
+      await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
+      await trx.rawQuery(
+        `UPDATE releases t SET
+           files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
+           first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
+           posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
+           first_seen_at = least(t.first_seen_at, s.first_seen_at), last_seen_at = greatest(t.last_seen_at, s.last_seen_at),
+           readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
+           file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
+           name = coalesce(nullif(?, ''), t.name)
+         FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
+                 min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
+                 max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at, max(last_seen_at) AS last_seen_at,
+                 bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
+                 max(file_total) AS file_total, max(inner_name) AS inner_name
+               FROM releases WHERE id = ANY(?)) s
+         WHERE t.id = ?`,
+        [name ?? '', others, target.id]
+      )
+      // Completeness as the settler decides it, once settled.
+      await trx.rawQuery(
+        `UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
+        [target.id]
+      )
+      await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
+      return others.length
+    }
+  }
+
+  /** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
+  private async setName(trx: any, setId: string): Promise<string | undefined> {
+    const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
+    const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
+    // A par2 made after the renaming names the random names again.
+    return name && !looksRandom(name) ? name.slice(0, 255) : undefined
+  }
+}

app/services/par2_pruner.ts

@@ -0,0 +1,81 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
+const SETTLED_MS = 2 * 86400_000
+/** Sets per statement. */
+const BATCH = 20_000
+/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
+const CURSOR = 'par2-prune'
+/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
+const JUNK = ['dead', 'fake', 'repeated']
+
+export interface PruneRound {
+  complete: number
+  judged: number
+  rows: number
+}
+
+/**
+ * Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
+ * 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
+ * done when
+ * - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
+ *   set (all of it merged into one);
+ * - judged: every release holding it has a junk verdict (dead, fake, repeated).
+ * par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
+ * par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
+ */
+export class Par2Pruner {
+  async round(): Promise<PruneRound> {
+    const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
+    const saved = await db.from('worker_cursors').where('name', CURSOR).first()
+    let since: Date = saved ? new Date(saved.at) : new Date(0)
+    const until = new Date(Date.now() - SETTLED_MS)
+    for (;;) {
+      const { rows: sets } = await db.rawQuery(
+        `SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
+         HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
+        [since, until, BATCH]
+      )
+      if (sets.length === 0) break
+      const ids = sets.map((s: { set_id: string }) => s.set_id)
+      const { rows: done } = await db.rawQuery(
+        `SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
+         WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
+           AND NOT EXISTS (
+             SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
+               SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
+               WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
+        [ids]
+      )
+      round.complete += done.length
+      round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
+      since = sets.at(-1).at
+      await db.rawQuery(
+        `INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
+        [CURSOR, since]
+      )
+      if (sets.length < BATCH) break
+    }
+    // Judged: no cursor (verdicts come and go); only sets with rows left.
+    const { rows: judged } = await db.rawQuery(
+      `SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
+       WHERE v.rule = ANY(?)
+         AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
+         AND NOT EXISTS (
+           SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
+           WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
+      [JUNK, JUNK]
+    )
+    round.judged = judged.length
+    const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
+    for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
+    return round
+  }
+
+  private async drop(setIds: string[]): Promise<number> {
+    if (setIds.length === 0) return 0
+    const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
+    return result.rowCount ?? 0
+  }
+}

app/services/planner.ts

@@ -0,0 +1,145 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+import NntpPool from '#services/nntp_pool'
+
+/**
+ * The indexed groups and their ranges (REBUILD.md, group list, coordinator).
+ * New groups get priorities after the existing ones (owner rule). A group's
+ * span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is
+ * its position in its group's history (0 oldest … 1000 newest), so all groups
+ * start with their oldest era. Planning again only adds ranges for new articles.
+ */
+class Planner {
+  async addGroups(names: string[]): Promise<number> {
+    let added = 0
+    for (const name of names) {
+      const { rows } = await db.rawQuery(
+        `INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`,
+        [name]
+      )
+      added += rows.length
+    }
+    return added
+  }
+
+  /** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */
+  async largestGroups(pattern: string, top: number): Promise<string[]> {
+    const groups = await NntpPool.use((client) => client.listActive(pattern))
+    return groups
+      .filter((g) => g.high > g.low)
+      .sort((a, b) => b.high - b.low - (a.high - a.low))
+      .slice(0, top)
+      .map((g) => g.name)
+  }
+
+  async planGroup(group: { id: number; name: string }): Promise<number> {
+    const info = await NntpPool.use((client) => client.group(group.name))
+    const size = coordinatorConfig.rangeArticles
+    const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
+    const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
+    const span = Math.max(1, info.high - info.low)
+    const rows = []
+    for (let start = from; start <= info.high; start += size) {
+      rows.push({
+        group_id: group.id,
+        start_id: start,
+        end_id: Math.min(start + size - 1, info.high),
+        next_id: start,
+        priority: Math.floor((1000 * (start - info.low)) / span),
+      })
+    }
+    for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
+    await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
+    return rows.length
+  }
+
+  /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
+  async planAll(): Promise<{ groups: number; ranges: number }> {
+    const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
+    let ranges = 0
+    for (const group of groups) {
+      // One group failing (e.g. gone from the server) doesn't stop the others.
+      try {
+        ranges += await this.planGroup({ id: Number(group.id), name: group.name })
+      } catch (error: any) {
+        logger.warn(`Planning ${group.name}: ${error.message}`)
+      }
+    }
+    return { groups: groups.length, ranges }
+  }
+
+  /**
+   * The live tail: ranges for the articles after each planned group's last
+   * range, at the tail's priority (before any history). The newest article
+   * numbers come from LIST ACTIVE, one command per name prefix (GROUP per
+   * group took minutes for 1000 groups).
+   */
+  async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
+    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
+    if (groups.length === 0) return { groups: 0, ranges: 0 }
+    const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
+    const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
+    const highs = new Map<string, number>()
+    await NntpPool.use(async (client) => {
+      for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
+        for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
+      }
+    })
+    const rows = tailRanges(
+      groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
+      now,
+      { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
+    )
+    if (rows.length === 0) return { groups: 0, ranges: 0 }
+    await db.transaction(async (trx) => {
+      await trx.table('ranges').multiInsert(rows)
+      for (const id of new Set(rows.map((r) => r.group_id))) {
+        const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
+        await trx.from('groups').where('id', id).update({ high, planned_at: now })
+      }
+    })
+    return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
+  }
+}
+
+/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
+export function wildmatOf(name: string): string {
+  const parts = name.split('.')
+  return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
+}
+
+export interface TailGroup {
+  id: number
+  /** End of its last range (null: none, then it has no tail yet). */
+  plannedTo: number | null
+  /** When its last range was planned. */
+  plannedAt: Date
+  /** The server's newest article (null: not listed). */
+  high: number | null
+}
+
+/**
+ * The tail's new ranges: per group, the articles after its last range, in
+ * ranges of at most `rangeArticles`, once there are `minArticles` of them or
+ * the group's last planning is `maxDelayMinutes` ago.
+ */
+export function tailRanges(
+  groups: TailGroup[],
+  now: Date,
+  options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
+) {
+  const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
+  for (const g of groups) {
+    if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
+    const waiting = g.high - g.plannedTo
+    const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
+    if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
+    for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
+      rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
+    }
+  }
+  return rows
+}
+
+export default new Planner()

app/services/prefetcher.ts

@@ -0,0 +1,439 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import coordinatorConfig from '#config/coordinator'
+import NntpPool from '#services/nntp_pool'
+import Spool, { type ChunkSpan } from '#services/spool'
+import Corpus from '#services/corpus'
+import CatalogSpace from '#services/catalog_space'
+import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
+
+interface Target {
+  id: number
+  group: string
+  /** Chunk starts to fetch without compression (ranges.plain_chunks). */
+  plain: Set<number>
+  /** Chunks not spooled yet, in order. */
+  missing: ChunkSpan[]
+  /** All of the range's chunks, from its start. */
+  total: number
+  start: number
+  end: number
+  /** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
+  capture: boolean
+  /** Fetched into the corpus only, not the spool: a capture range no worker leases soon (corpus.ahead). */
+  corpusOnly?: boolean
+}
+
+/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
+const STALL_SAMPLE_MS = 15_000
+const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS
+
+/** Connection ages the report counts slow first requests by, in minutes. */
+const AGE_BUCKETS = [2, 10, Infinity]
+
+/** How often the targets are read again from the database. */
+const REFRESH_MS = 2000
+/** How often the spool's size is measured (a stat per file). */
+const SIZE_MS = 30_000
+
+/**
+ * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
+ * Every connection takes the next missing chunk: the leased ranges' first
+ * (a worker waits for them), then the next planned ranges, with at most
+ * PREFETCH_PER_RANGE connections on a range and PREFETCH_PER_GROUP on a
+ * group (Eweka answers faster spread out), about SPOOL_AHEAD_PER_LEASE per
+ * active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
+ * connection fills ~26 k overview lines/s, so a range on one connection
+ * kept its worker waiting (LOG.md 2026-10-09). A chunk that takes long is
+ * asked again on a spare connection (fetchHedged).
+ */
+export class Prefetcher {
+  /**
+   * This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
+   * process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
+   * LOG.md 2026-10-09), less than 50 connections bring.
+   */
+  constructor(private shard = { index: 0, count: 1 }) {}
+
+  private targets: Target[] = []
+  private claimed = new Set<string>()
+  /** Chunks being fetched per range and per group (spool.perRange, spool.perGroup). */
+  private perRange = new Map<number, number>()
+  private perGroup = new Map<string, number>()
+  private running = 0
+  private refreshedAt = 0
+  private stopped = false
+  private spoolBytes = 0
+  private sizedAt = 0
+  /** Fetched since the last report: chunks, articles, ms spent fetching. */
+  private stats = this.freshStats()
+  /** Loops in a fetch or a spool write right now. */
+  private fetchingNow = 0
+  private writingNow = 0
+  private sampler?: NodeJS.Timeout
+  /** Requests in flight: what, since when, on which connection (the report names the oldest). */
+  private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
+  /** Capture ranges found complete in the corpus (not looked at again). */
+  private stored = new Set<number>()
+
+  private freshStats() {
+    return {
+      chunks: 0,
+      /** Chunks read from the corpus instead of fetched. */
+      stored: 0,
+      /** Chunks fetched into the corpus only (corpus.ahead). */
+      captured: 0,
+      articles: 0,
+      fetchMs: 0,
+      hedged: 0,
+      hedgeWon: 0,
+      /** First requests per connection-age bucket, and how many of them were slow. */
+      byAge: AGE_BUCKETS.map(() => ({ requests: 0, slow: 0 })),
+      /** Spool writes and the range's update, ms. */
+      writeMs: 0,
+      /** Per answered request, ms: GROUP, XOVER until its status line, the transfer; transfers with a silence over 5 s. */
+      requests: 0,
+      groupMs: 0,
+      replyMs: 0,
+      bodyMs: 0,
+      silent: 0,
+      /** Once a second: loops fetching, writing, not running; pool connections with a command in flight. */
+      samples: 0,
+      fetching: 0,
+      writing: 0,
+      idle: 0,
+      inFlight: 0,
+      inFlightMin: Infinity,
+      /** Requests given up as stalled (nntpStallBytesPerMinute). */
+      deadlines: 0,
+      ranges: 0,
+      groups: 0,
+      since: Date.now(),
+    }
+  }
+
+  /** The prefetcher's loops: the pool less the connections kept for hedging. */
+  private get loops() {
+    return Math.max(1, NntpPool.size - coordinatorConfig.nntpHedgeConnections)
+  }
+
+  /** One round: refreshes the targets now and then, and puts idle connections to work. */
+  async tick() {
+    this.sampler ??= setInterval(() => this.sample(), 1000).unref()
+    if (Date.now() - this.refreshedAt >= REFRESH_MS) {
+      this.refreshedAt = Date.now()
+      this.targets = await this.readTargets()
+    }
+    this.report()
+    while (!this.stopped && this.running < this.loops && this.peek()) {
+      this.running++
+      this.work()
+        .catch((error) => logger.warn(`Prefetching: ${error.message}`))
+        .finally(() => this.running--)
+    }
+  }
+
+  stop() {
+    this.stopped = true
+    clearInterval(this.sampler)
+  }
+
+  get active() {
+    return this.running
+  }
+
+  /** One connection's loop: chunks until there is nothing left to fetch. */
+  private async work() {
+    while (!this.stopped) {
+      const next = this.take()
+      if (!next) return
+      const { target, span } = next
+      try {
+        const fetchedAt = Date.now()
+        this.fetchingNow++
+        const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
+          await client.group(target.group)
+          return client.xoverRaw(span.from, span.to)
+        }
+        const corpusRange = { group: target.group, start: target.start, end: target.end }
+        // A range in the corpus is read from disk (a reset re-indexes it without the news server).
+        let data = target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
+        const stored = data !== null
+        try {
+          if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
+        } finally {
+          this.fetchingNow--
+        }
+        if (stored) {
+          this.stats.stored++
+        } else {
+          this.stats.chunks++
+          this.stats.articles += span.to - span.from + 1
+          this.stats.fetchMs += Date.now() - fetchedAt
+          if (target.capture) await Corpus.write(corpusRange, span, data, target.total)
+        }
+        if (target.corpusOnly) {
+          this.stats.captured++
+          continue
+        }
+        const writtenAt = Date.now()
+        this.writingNow++
+        try {
+          const existed = await Spool.has(target.id, span)
+          await Spool.write(target.id, span, data)
+          if (existed) continue
+          this.spoolBytes += data.length
+          const { rows } = await db.rawQuery(
+            `UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
+            [target.id]
+          )
+          if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
+        } finally {
+          this.writingNow--
+          this.stats.writeMs += Date.now() - writtenAt
+        }
+      } finally {
+        this.claimed.delete(`${target.id}:${span.from}`)
+        this.count(target, -1)
+      }
+    }
+  }
+
+  /**
+   * A chunk over a pooled connection. Not in after nntpHedgeAfterMs, it's
+   * asked again on a free connection (if there is one): Eweka stalls single
+   * requests, and the same chunk on another connection mostly comes in
+   * seconds (LOG.md 2026-10-09 (15)). The first answer wins; the other
+   * request can't be cancelled, so its connection is closed (the pool opens
+   * a fresh one).
+   */
+  private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
+    let age: number | undefined
+    const request = (start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null, first: boolean): Request<Buffer> | null => {
+      let client: XoverClient | undefined
+      let cancelled = false
+      const done = start(async (c) => {
+        // Given up before it got a connection: the connection goes back unused.
+        if (cancelled) return Buffer.alloc(0)
+        client = c
+        if (first) age = NntpPool.ageOf(c)
+        const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
+        this.inFlight.add(entry)
+        // Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall.
+        let late = false
+        const samples: number[] = []
+        const watch = setInterval(() => {
+          samples.push(c.buffered)
+          if (samples.length <= STALL_SAMPLES) return
+          const minuteAgo = samples.shift()!
+          if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) {
+            late = true
+            this.stats.deadlines++
+            c.close()
+          }
+        }, STALL_SAMPLE_MS)
+        let data: Buffer
+        let groupMs = 0
+        try {
+          const groupAt = performance.now()
+          await c.group(group)
+          groupMs = performance.now() - groupAt
+          data = await c.xoverRaw(span.from, span.to)
+        } catch (error) {
+          if (!late) throw error
+          throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`)
+        } finally {
+          clearInterval(watch)
+          this.inFlight.delete(entry)
+        }
+        if (!cancelled) {
+          const t = c.lastTiming
+          this.stats.requests++
+          this.stats.groupMs += groupMs
+          this.stats.replyMs += t.replyMs
+          this.stats.bodyMs += t.bodyMs
+          if (t.longestGapMs > 5000) this.stats.silent++
+        }
+        return data
+      })
+      if (!done) return null
+      done.catch(() => {})
+      return {
+        done,
+        // The request can't be called off on the wire: its connection is closed, and the pool opens a fresh one.
+        cancel: () => {
+          cancelled = true
+          client?.close()
+        },
+      }
+    }
+    const result = await hedge(
+      () => request((work) => NntpPool.use(work), true)!,
+      () => request((work) => NntpPool.tryUse(work), false),
+      coordinatorConfig.nntpHedgeAfterMs
+    )
+    if (age !== undefined) {
+      const bucket = this.stats.byAge[AGE_BUCKETS.findIndex((minutes) => age! < minutes * 60_000)]
+      bucket.requests++
+      if (result.late) bucket.slow++
+    }
+    if (result.hedged) this.stats.hedged++
+    if (result.secondWon) this.stats.hedgeWon++
+    return result.value
+  }
+
+  /** The longest-running request: range, chunk, how long, how much came in. */
+  private oldest() {
+    let oldest: { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
+    for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
+    if (!oldest) return 'none in flight'
+    return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
+  }
+
+  private sample() {
+    const s = this.stats
+    s.samples++
+    s.fetching += this.fetchingNow
+    s.writing += this.writingNow
+    s.idle += Math.max(0, this.loops - this.fetchingNow - this.writingNow)
+    s.inFlight += NntpPool.busyCount
+    s.inFlightMin = Math.min(s.inFlightMin, NntpPool.busyCount)
+    s.ranges += this.perRange.size
+    s.groups += this.perGroup.size
+  }
+
+  /** A line a minute: what the connections fetched, and how long a chunk took. */
+  private report() {
+    const seconds = (Date.now() - this.stats.since) / 1000
+    if (seconds < 60) return
+    const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
+    if (chunks > 0) {
+      const ages = byAge
+        .map((b, i) => `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace('– min', '+ min'))
+        .join(', ')
+      logger.info(
+        `Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
+          `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
+          `hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
+          (this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus` : '') +
+          (this.stats.captured > 0 ? `; ${this.stats.captured} fetched into the corpus only` : '') +
+          '.'
+      )
+      const st = this.stats
+      const avg = (n: number) => (samples > 0 ? n / samples : 0)
+      const per = (ms: number) => (requests > 0 ? (ms / requests / 1000).toFixed(1) : '-')
+      logger.info(
+        `Connections: ${avg(st.inFlight).toFixed(1)} of ${NntpPool.size} with a command in flight (min ${st.inFlightMin}), ` +
+          `on ${avg(st.ranges).toFixed(1)} ranges of ${avg(st.groups).toFixed(1)} groups; ` +
+          `loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` +
+          `a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` +
+          `${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` +
+          `${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.`
+      )
+    }
+    this.stats = this.freshStats()
+  }
+
+  private peek(): boolean {
+    return this.targets.some((t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
+  }
+
+  /** Whether the target is under its range's and its group's limit of connections. */
+  private open(target: Target) {
+    const { perRange, perGroup } = coordinatorConfig.spool
+    return (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) && (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
+  }
+
+  private count(target: Target, by: number) {
+    const range = (this.perRange.get(target.id) ?? 0) + by
+    const group = (this.perGroup.get(target.group) ?? 0) + by
+    if (range > 0) this.perRange.set(target.id, range)
+    else this.perRange.delete(target.id)
+    if (group > 0) this.perGroup.set(target.group, group)
+    else this.perGroup.delete(target.group)
+  }
+
+  /** The first unclaimed missing chunk, in target order, of a target under its limits. */
+  private take(): { target: Target; span: ChunkSpan } | null {
+    for (const target of this.targets) {
+      if (!this.open(target)) continue
+      const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
+      if (index === -1) continue
+      const [span] = target.missing.splice(index, 1)
+      this.claimed.add(`${target.id}:${span.from}`)
+      this.count(target, 1)
+      return { target, span }
+    }
+    return null
+  }
+
+  private async readTargets(): Promise<Target[]> {
+    if (Date.now() - this.sizedAt >= SIZE_MS) {
+      this.sizedAt = Date.now()
+      this.spoolBytes = await Spool.size()
+    }
+    const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
+    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
+    // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
+    const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
+    // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
+    const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
+    const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
+    const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
+    const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
+    // With at most perRange connections a range, enough ranges still being fetched for all loops: fully
+    // spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
+    // 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
+    const { perRange } = coordinatorConfig.spool
+    const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
+    const want = Math.max(ahead - Number(ready), short)
+    // The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
+    const full = await CatalogSpace.full()
+    const fresh = room && want > 0
+      ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').where((q: any) => (full ? q.where('r.priority', '<', 0) : q)).orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want)
+      : []
+    const targets: Target[] = []
+    for (const row of [...leased, ...(room ? started : []), ...fresh]) {
+      const start = Number(row.start_id)
+      const end = Number(row.end_id)
+      const total = Spool.chunksOf({ start, end }).length
+      const missing: ChunkSpan[] = []
+      for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
+        if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
+      }
+      if (missing.length > 0) {
+        targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
+      }
+    }
+    // Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
+    for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id)))) targets.push(target)
+    return targets
+  }
+
+  /** Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus. */
+  private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
+    const { dir, ahead } = coordinatorConfig.corpus
+    if (!dir || ahead <= 0) return []
+    const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
+    const targets: Target[] = []
+    for (const row of candidates) {
+      if (targets.length >= ahead) break
+      const id = Number(row.id)
+      if (taken.has(id) || this.stored.has(id)) continue
+      const start = Number(row.start_id)
+      const end = Number(row.end_id)
+      const range = { group: row.group, start, end }
+      if (await Corpus.complete(range)) {
+        this.stored.add(id)
+        continue
+      }
+      const chunks = Spool.chunksOf({ start, end })
+      const missing: ChunkSpan[] = []
+      for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
+      if (missing.length > 0) {
+        targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
+      }
+    }
+    return targets
+  }
+}

app/services/prober.ts

@@ -0,0 +1,297 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { setTimeout as sleep } from 'node:timers/promises'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { YencError } from '#lib/yenc/index'
+import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
+
+interface Candidate {
+  id: number
+  releaseId: number
+  kind: string
+  messageId: string
+}
+
+type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
+
+/**
+ * The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
+ * par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones).
+ */
+const TIERS: { kind: string; unreadable: boolean }[] = [
+  { kind: 'lone', unreadable: false },
+  { kind: 'par2', unreadable: true },
+  { kind: 'archive', unreadable: true },
+  { kind: 'par2', unreadable: false },
+  { kind: 'archive', unreadable: false },
+]
+const FLUSH_MS = 2000
+/** Slices a claim takes from random points of the queue. */
+const SLICES = 20
+const REPORT_MS = 60_000
+
+/**
+ * Phase 2's first step (REBUILD.md, part 6): reads the probe queue's
+ * articles (settled releases first, by kind) over PROBE_CONNECTIONS
+ * connections of its own, without compression, and stores what they show:
+ * per release and kind an Inspection, and on the release whether it's
+ * passworded and its inner name. One process: fetching (~0.1 s an article)
+ * dwarfs parsing (a few ms), so there's no queue between them.
+ */
+export class Prober {
+  private queue: Candidate[] = []
+  private outcomes: Outcome[] = []
+  private refilling: Promise<void> | null = null
+  private stopped = false
+  private stats = Prober.freshStats()
+
+  /** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
+  private static freshStats() {
+    return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
+  }
+
+  async run() {
+    // A previous run's claims: one prober at a time.
+    const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
+    if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
+    const connections = coordinatorConfig.probe.connections
+    logger.info(`Probing with ${connections} connections.`)
+    const flusher = (async () => {
+      while (!this.stopped || this.outcomes.length > 0) {
+        await sleep(FLUSH_MS)
+        const flushed = Date.now()
+        await this.flush().catch((error) => logger.error(`Probing: storing results failed: ${error.message}`))
+        this.stats.flushMs += Date.now() - flushed
+        this.report()
+      }
+    })()
+    await Promise.all(Array.from({ length: connections }, () => this.lane()))
+    // What was claimed and not probed goes back.
+    for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
+    this.stopped = true
+    await flusher
+  }
+
+  stop() {
+    this.stopped = true
+  }
+
+  /** One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure. */
+  private async lane() {
+    let client: XoverClient | null = null
+    while (!this.stopped) {
+      const batch: Candidate[] = []
+      const waited = Date.now()
+      while (batch.length < coordinatorConfig.probe.pipeline) {
+        const candidate = await this.next()
+        if (!candidate) break
+        batch.push(candidate)
+      }
+      this.stats.waitMs += Date.now() - waited
+      if (batch.length === 0) {
+        this.stats.idleMs += 30_000
+        await sleep(30_000)
+        continue
+      }
+      try {
+        client ??= await XoverClient.connect({
+          host: env.get('NNTP_HOST')!,
+          port: env.get('NNTP_PORT', 563),
+          user: env.get('NNTP_USER'),
+          password: env.get('NNTP_PASSWORD'),
+          secure: env.get('NNTP_SECURE', true),
+          compress: false,
+          timeoutMs: coordinatorConfig.nntpTimeoutMs,
+        })
+        const fetched = Date.now()
+        const bodies = await client.bodies(batch.map((c) => c.messageId))
+        this.stats.fetchMs += Date.now() - fetched
+        const inspected = Date.now()
+        for (const [i, candidate] of batch.entries()) {
+          const body = bodies[i]
+          this.stats.articles++
+          if (body === null) {
+            // Taken down or expired: final.
+            this.outcomes.push({ candidate, status: 'missing' })
+            continue
+          }
+          this.stats.bytes += body.length
+          try {
+            this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) })
+          } catch (error: any) {
+            // Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
+            if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
+            this.outcomes.push({ candidate, status: 'error' })
+          }
+        }
+        this.stats.inspectMs += Date.now() - inspected
+      } catch (error: any) {
+        logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
+        for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
+        client?.close()
+        client = null
+        await sleep(5000)
+      }
+    }
+    client?.close()
+  }
+
+  private async next(): Promise<Candidate | undefined> {
+    if (this.queue.length < coordinatorConfig.probe.batch / 2) {
+      this.refilling ??= this.claim().finally(() => (this.refilling = null))
+      if (this.queue.length === 0) await this.refilling
+    }
+    return this.queue.shift()
+  }
+
+  /**
+   * Waiting candidates of settled releases, by kind, claimed so another round
+   * doesn't take them: SLICES slices from random points of the queue. In id
+   * order it went group by group, and some groups' articles take 0.6–3 s
+   * where most take 20–60 ms (a.b.zogg 2016: 7 articles/s on 6 connections).
+   */
+  private async claim() {
+    const started = Date.now()
+    try {
+      await this.claimTiers()
+    } finally {
+      this.stats.claimMs += Date.now() - started
+    }
+  }
+
+  private async claimTiers() {
+    for (const { kind, unreadable } of TIERS) {
+      const want = coordinatorConfig.probe.batch - this.queue.length
+      if (want <= 0) return
+      // The starting points chosen here: in the query Postgres drew random() once for all slices.
+      // Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
+      const [{ lo, hi }] = (
+        await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable >= ?`, [kind, unreadable])
+      ).rows
+      if (lo === null) continue
+      const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
+      const { rows } = await db.rawQuery(
+        `UPDATE probe_candidates SET status = 'taken' WHERE id IN (
+           SELECT pc.id FROM unnest(?::bigint[]) AS o(start)
+           CROSS JOIN LATERAL (
+             SELECT pc.id FROM probe_candidates pc
+             WHERE pc.status = 'waiting' AND pc.ready AND pc.kind = ? AND pc.unreadable >= ? AND pc.id >= o.start
+             ORDER BY pc.kind, pc.unreadable, pc.id LIMIT ? FOR UPDATE SKIP LOCKED) pc)
+         RETURNING id, release_id, kind, message_id`,
+        [starts, kind, unreadable, Math.ceil(want / SLICES)]
+      )
+      const claimed = rows.map((r: any) => ({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id }))
+      // Probed before (an earlier catalog's candidate of the same article): answered from probe_cache.
+      const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
+        claimed.map((c: Candidate) => c.messageId),
+        PROBE_VERSION,
+      ])
+      const known = new Map(cached.map((c: any) => [c.message_id, c]))
+      for (const candidate of claimed) {
+        const hit: any = known.get(candidate.messageId)
+        if (hit) {
+          this.stats.cached++
+          this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
+        } else {
+          this.queue.push(candidate)
+        }
+      }
+    }
+  }
+
+  /** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
+  private async flush() {
+    const outcomes = this.outcomes.splice(0)
+    if (outcomes.length === 0) return
+    for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++
+    await db.transaction(async (trx) => {
+      await trx.rawQuery(
+        `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
+        [JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
+      )
+      // Final answers read from the news server, kept by message-id for the next catalog (a reset).
+      const fresh = outcomes.filter((o) => !o.cached && o.status !== 'waiting')
+      if (fresh.length > 0) {
+        await trx.rawQuery(
+          `INSERT INTO probe_cache (message_id, status, result, version)
+           SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
+           ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
+          [PROBE_VERSION, JSON.stringify(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
+        )
+      }
+      const done = outcomes.filter((o) => o.inspection)
+      if (done.length === 0) return
+      await trx.rawQuery(
+        `INSERT INTO release_inspections (release_id, candidate_id, kind, result)
+         SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
+         ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
+        // A par2 set's file list goes to par2_files, not into every inspection.
+        [JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
+      )
+      await this.storeForMerge(trx, done)
+      // Each touched release from all its inspections (earlier rounds may have stored others).
+      const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
+      const { rows } = await trx.rawQuery(
+        `SELECT i.release_id, i.kind, i.result, r.name FROM release_inspections i JOIN releases r ON r.id = i.release_id WHERE i.release_id = ANY(?)`,
+        [ids]
+      )
+      const byRelease = new Map<number, { name: string; all: { kind: string; result: Inspection }[] }>()
+      for (const r of rows) {
+        const id = Number(r.release_id)
+        const entry = byRelease.get(id) ?? { name: r.name as string, all: [] as { kind: string; result: Inspection }[] }
+        entry.all.push({ kind: r.kind, result: r.result })
+        byRelease.set(id, entry)
+      }
+      const updates = [...byRelease].map(([id, { name, all }]) => ({
+        id,
+        passworded: all.some((i) => i.result.passworded === true),
+        inner_name: innerName(all, name),
+      }))
+      await trx.rawQuery(
+        `UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
+         FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
+        [JSON.stringify(updates)]
+      )
+    })
+  }
+
+  /** What the par2 merge matches (app/services/par2_merger.ts): files' first-16-KiB hashes, par2 sets' file lists. */
+  private async storeForMerge(trx: any, done: Outcome[]) {
+    const hashed = done.filter((o) => o.inspection!.hash16k)
+    if (hashed.length > 0) {
+      await trx.rawQuery(
+        `INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
+         SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
+         ON CONFLICT (candidate_id) DO NOTHING`,
+        [JSON.stringify(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
+      )
+    }
+    for (const o of done) {
+      const set = o.inspection!.par2
+      if (!set) continue
+      await trx.rawQuery(
+        `INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
+         SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
+         ON CONFLICT DO NOTHING`,
+        [set.setId, JSON.stringify(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
+      )
+      await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
+    }
+  }
+
+  private report() {
+    const seconds = (Date.now() - this.stats.since) / 1000
+    if (seconds < REPORT_MS / 1000) return
+    const { articles, bytes, done, missing, error } = this.stats
+    logger.info(
+      `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
+        `${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache. ` +
+        `Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
+        `idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
+        `claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
+    )
+    this.stats = Prober.freshStats()
+  }
+}

app/services/range_blocks.ts

@@ -0,0 +1,31 @@
+import db from '@adonisjs/lucid/services/db'
+
+/**
+ * range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
+ * release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
+ * range that isn't done or doesn't start where the one before ended.
+ */
+class RangeBlocks {
+  async rebuild(): Promise<number> {
+    return db.transaction(async (trx) => {
+      await trx.rawQuery(`DELETE FROM range_blocks`)
+      const result = await trx.rawQuery(
+        `WITH r AS (
+           SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
+                  min(start_id) OVER (PARTITION BY group_id) AS group_first,
+                  lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
+           FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
+         runs AS (
+           SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
+                       OVER (PARTITION BY group_id ORDER BY start_id) AS run
+           FROM r)
+         INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
+         SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
+         FROM runs WHERE done GROUP BY group_id, run`
+      )
+      return result.rowCount ?? 0
+    })
+  }
+}
+
+export default new RangeBlocks()

app/services/release_judge.ts

@@ -0,0 +1,159 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Raised when a rule changes: verdicts of an older version are judged again. */
+export const JUDGE_VERSION = 1
+/** Between rounds: each rule is a scan of its table. */
+const ROUND_MS = 60 * 60 * 1000
+/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
+const REPEATED_RELEASES = 20
+
+/**
+ * A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
+ * these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
+ * 2026-10-16 (64)), so {0,1}.
+ */
+const MEDIA =
+  `(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
+/** The posters of scraped celebrity pictures. */
+const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
+/** An executable's name. */
+const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
+
+export interface Rule {
+  name: string
+  /** Why, in a few words (the log, the web interface). */
+  says: string
+  /** SELECT of the release ids the rule judges. */
+  sql: string
+}
+
+/**
+ * The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
+ * the header pass, the prober, the merges and the spots know by now.
+ */
+export const RULES: Rule[] = [
+  {
+    name: 'dead',
+    says: 'every probed first article is gone from the server (taken down or expired)',
+    sql: `SELECT release_id FROM probe_candidates WHERE status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(status = 'missing')`,
+  },
+  {
+    name: 'fake',
+    says: 'named as a film, episode or music, but the content is a program',
+    // An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
+    // an executable by name or by its first bytes, under a media name.
+    sql: `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
+          WHERE i.kind = 'archive' AND jsonb_array_length(i.result->'names') > 0
+            AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
+            AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
+          UNION
+          SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id
+          WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
+            AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
+  },
+  {
+    name: 'repeated',
+    says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
+    sql: `SELECT DISTINCT h.release_id FROM file_hashes h
+          JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
+  },
+  {
+    name: 'scraped',
+    says: 'celebrity pictures scraped from the web, reposted by one spam family',
+    // "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
+    // a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
+    // titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
+    sql: `SELECT r.id FROM releases r
+          WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
+  },
+  {
+    name: 'locked',
+    says: 'passworded, and no spot, NZB or title gives the password',
+    sql: `SELECT r.id FROM releases r WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
+  },
+  {
+    name: 'unusable',
+    says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
+    // Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
+    sql: `SELECT r.id FROM releases r
+          WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
+            AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
+            AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
+            AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
+            AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
+  },
+]
+
+export interface JudgeRound {
+  /** Per rule: releases it judges now, verdicts added, verdicts taken back. */
+  rules: Record<string, { matched: number; added: number; removed: number }>
+}
+
+/**
+ * The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
+ * signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Each
+ * round recomputes every rule: a release a rule matches gets a verdict (the first rule in order, over a later one's), one it no longer
+ * matches loses its verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
+ */
+export class ReleaseJudge {
+  private stopped = false
+
+  stop() {
+    this.stopped = true
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        const round = await this.round()
+        const parts = Object.entries(round.rules).map(([n, r]) => `${n} ${r.matched} (+${r.added} −${r.removed})`)
+        logger.info(`Judged: ${parts.join(', ')}.`)
+      } catch (error: any) {
+        logger.error(`Judge round failed: ${error.message}`)
+      }
+      for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  /** One round over every rule; with dry, only counts. */
+  async round(dry = false): Promise<JudgeRound> {
+    const round: JudgeRound = { rules: {} }
+    // Verdicts of an older rule set are judged again from scratch.
+    if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
+    const judged = new Set<string>()
+    for (const rule of RULES) {
+      if (this.stopped) break
+      const { rows } = await db.rawQuery(rule.sql)
+      const ids = rows.map((r: Record<string, unknown>) => String(Object.values(r)[0])).filter((id: string) => !judged.has(id))
+      for (const id of ids) judged.add(id)
+      const stats = { matched: ids.length, added: 0, removed: 0 }
+      if (!dry) {
+        await db.transaction(async (trx) => {
+          // Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
+          await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
+          await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
+          for (let i = 0; i < ids.length; i += 50_000) {
+            await trx.rawQuery(`INSERT INTO judged SELECT unnest(?::bigint[]) ON CONFLICT DO NOTHING`, [ids.slice(i, i + 50_000)])
+          }
+          const removed = await trx.rawQuery(
+            `DELETE FROM release_verdicts v WHERE v.rule = ? AND NOT EXISTS (SELECT 1 FROM judged j WHERE j.release_id = v.release_id)`,
+            [rule.name]
+          )
+          const added = await trx.rawQuery(
+            `INSERT INTO release_verdicts (release_id, rule, rule_version)
+             SELECT j.release_id, ?, ? FROM judged j JOIN releases r ON r.id = j.release_id
+             ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
+             WHERE release_verdicts.rule <> excluded.rule`,
+            [rule.name, JUDGE_VERSION]
+          )
+          stats.removed = removed.rowCount ?? 0
+          stats.added = added.rowCount ?? 0
+        })
+      }
+      round.rules[rule.name] = stats
+    }
+    return round
+  }
+}

app/services/rule_sets.ts

@@ -0,0 +1,39 @@
+import db from '@adonisjs/lucid/services/db'
+import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
+
+type StoredRules = Omit<Rules, 'version' | 'priorities'>
+
+/**
+ * The rule set workers get with a lease (REBUILD.md, rules and recovery):
+ * the newest version, plus the indexed groups' priorities (owner rule).
+ * Version 1 (DEFAULT_RULES, no dropped signatures) is created when there is none.
+ */
+class RuleSets {
+  async current(): Promise<Rules> {
+    let row = await db.from('rule_sets').orderBy('version', 'desc').first()
+    if (!row) {
+      const { version: _version, ...rules } = DEFAULT_RULES
+      await db.table('rule_sets').insert({ version: 1, rules: JSON.stringify({ ...rules, droppedSignatures: [] }), note: 'default' }).onConflict('version').ignore()
+      row = await db.from('rule_sets').orderBy('version', 'desc').first()
+    }
+    const groups = await db.from('groups').where('active', true).select('name', 'priority')
+    const { version: _v, ...defaults } = DEFAULT_RULES
+    return {
+      // A field newer than the stored set takes today's default.
+      ...defaults,
+      ...(row.rules as StoredRules),
+      version: Number(row.version),
+      priorities: Object.fromEntries(groups.map((g) => [g.name, Number(g.priority)])),
+    }
+  }
+
+  /** A new version: what changes is recorded on the ranges that run with it. */
+  async publish(rules: StoredRules, note?: string): Promise<number> {
+    const latest = await db.from('rule_sets').max('version as v').first()
+    const version = Number(latest?.v ?? 0) + 1
+    await db.table('rule_sets').insert({ version, rules: JSON.stringify(rules), note: note ?? null })
+    return version
+  }
+}
+
+export default new RuleSets()

app/services/search_service.ts

@@ -0,0 +1,317 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import NzbService from '#services/nzb_service'
+
+export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
+
+/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
+export interface SearchParams {
+  q?: string
+  /** Also releases the junk judge hid (release_verdicts). */
+  judged?: boolean
+  /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
+  readable?: boolean
+  /** Only releases known to have all their files. */
+  complete?: boolean
+  /** Leave out releases known to miss files (Newznab). */
+  notIncomplete?: boolean
+  /** Leave out releases the prober found passworded (encrypted archive entries or headers). */
+  unlocked?: boolean
+  /** How the release was recognised: stem, subject or post. */
+  source?: string
+  poster?: string
+  /** Exact category ids (already expanded from parents). */
+  categories?: number[]
+  maxAgeDays?: number
+  /** TV: season and episode, matched as S01E02 or 1x02 in the name. */
+  season?: number
+  episode?: number
+  /** Size bounds in bytes. */
+  minSize?: number
+  maxSize?: number
+  sort?: SortOrder
+  page?: number
+  perPage?: number
+  /** Overrides `page` (Newznab pages by offset). */
+  offset?: number
+  /** Count matches up to this many past the page; `total` is capped there. */
+  countLimit?: number
+}
+
+/** A release in search results. */
+export interface ReleaseSummary {
+  id: number
+  name: string
+  source: string
+  poster: string | null
+  groups: string[]
+  postedAt: Date | null
+  fileTotal: number | null
+  fileCount: number
+  size: number
+  readable: boolean
+  complete: boolean | null
+  /** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
+  final: boolean
+  category: number
+  settled: boolean
+  /** The prober found encrypted archive entries or headers; null: not probed (yet). */
+  passworded: boolean | null
+  /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
+  innerName: string | null
+}
+
+/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
+const MAX_POSTERS = 200
+const SORTS: Record<SortOrder, string> = {
+  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
+  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
+  largest: 'r.bytes DESC, r.id DESC',
+  files: 'r.files DESC, r.id DESC',
+  name: 'r.name ASC, r.id ASC',
+}
+
+/** A broad search needn't count every match (each counted row is a heap read). */
+const COUNT_LIMIT = 10_000
+const STATS_TTL_MS = 5 * 60 * 1000
+/** A refresh that takes longer is given up (the next request tries again). */
+const STATS_TIMEOUT_MS = 2 * 60 * 1000
+
+/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
+function contains(word: string): string {
+  return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
+}
+
+/** Searching and browsing the catalog, for the web interface and Newznab. */
+class SearchService {
+  async search(params: SearchParams) {
+    const where: string[] = ['r.files > 0']
+    // Judged junk (app/services/release_judge.ts) is left out unless asked for.
+    if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
+    const bindings: unknown[] = []
+    // Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
+    for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
+      where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
+      bindings.push(contains(word), contains(word))
+    }
+    if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
+    if (params.complete) where.push('r.complete')
+    // Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
+    if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
+    if (params.unlocked) where.push('r.passworded IS NOT TRUE')
+    if (params.source) {
+      where.push('r.source = ?')
+      bindings.push(params.source)
+    }
+    // A poster search: the matching posters first (posters, by trigram), each one's releases then by
+    // releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
+    // poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
+    let posters: string[] | null = null
+    if (params.poster) {
+      const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
+      posters = matched.map((m: { poster: string }) => m.poster)
+    }
+    if (params.categories?.length) {
+      where.push('r.category = ANY(?)')
+      bindings.push(params.categories)
+    }
+    if (params.maxAgeDays) {
+      where.push('r.posted_at >= now() - make_interval(days => ?)')
+      bindings.push(params.maxAgeDays)
+    }
+    if (params.minSize !== undefined) {
+      where.push('r.bytes >= ?')
+      bindings.push(params.minSize)
+    }
+    if (params.maxSize !== undefined) {
+      where.push('r.bytes <= ?')
+      bindings.push(params.maxSize)
+    }
+    if (params.season !== undefined) {
+      const s = params.season
+      const e = params.episode
+      where.push('r.name ~* ?')
+      bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
+    }
+
+    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
+    const page = Math.max(params.page ?? 1, 1)
+    const offset = params.offset ?? (page - 1) * perPage
+    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
+    const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
+    if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
+    // Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
+    // bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
+    const source =
+      posters === null
+        ? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
+        : {
+            from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
+                     SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
+            where: 'true',
+            bindings: [posters] as unknown[],
+          }
+    const counting =
+      posters === null
+        ? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
+        : db.rawQuery(
+            `SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
+               SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
+            [posters, ...bindings]
+          )
+    const [{ rows }, { rows: counted }] = await Promise.all([
+      db.rawQuery(
+        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
+                release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
+         FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
+        [...source.bindings, ...bindings, perPage, offset]
+      ),
+      counting,
+    ])
+    return {
+      page,
+      perPage,
+      offset,
+      total: Math.min(Number(counted[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(counted[0].total) > countLimit,
+      releases: (rows as any[]).map((r): ReleaseSummary => ({
+        id: Number(r.id),
+        name: r.name,
+        source: r.source,
+        poster: r.poster,
+        groups: r.groups,
+        postedAt: r.posted_at,
+        fileTotal: r.file_total,
+        fileCount: Number(r.files),
+        size: Number(r.bytes),
+        readable: r.readable,
+        complete: r.complete,
+        final: r.final,
+        category: r.category,
+        settled: r.settled_at !== null,
+        passworded: r.passworded,
+        innerName: r.inner_name,
+      })),
+    }
+  }
+
+  /** A release with its files, read from its chunks. */
+  async release(id: number) {
+    const r = await db.from('releases').where('id', id).first()
+    if (!r) return null
+    const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
+      NzbService.files(id),
+      db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
+      db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
+    ])
+    return {
+      id: Number(r.id),
+      name: r.name,
+      source: r.source,
+      poster: r.poster,
+      groups: r.groups,
+      postedAt: r.posted_at,
+      fileTotal: r.file_total,
+      fileCount: Number(r.files),
+      size: Number(r.bytes),
+      segments: Number(r.segments),
+      readable: r.readable,
+      complete: r.complete,
+      final: finality[0].final,
+      hasPar2: r.has_par2,
+      category: r.category,
+      firstSeenAt: r.first_seen_at,
+      lastSeenAt: r.last_seen_at,
+      settledAt: r.settled_at,
+      passworded: r.passworded,
+      innerName: r.inner_name,
+      inspectedAt: r.inspected_at,
+      /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
+      inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
+      files: files.map((f) => ({
+        name: f.name,
+        subject: f.subject,
+        poster: f.poster,
+        postedAt: f.postedAt === null ? null : new Date(f.postedAt),
+        size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
+        parts: f.segments.length,
+      })),
+    }
+  }
+
+  /** Totals for the web interface, computed at most once a minute (callers in between share them). */
+  async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= Promise.race([
+        this.computeStats(),
+        new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
+      ])
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        .catch((error) => {
+          logger.warn(`Stats: ${error.message}`)
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates (a catalog of 100M+ rows can't be counted on every visit):
+   * the row count from Postgres's statistics, below a million rows exact;
+   * readable share, files and size from a sample of 5,000 rows; ranges
+   * counted exactly (a few hundred thousand rows).
+   */
+  private async computeStats() {
+    const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
+    const estimate = Number(rows[0]?.n ?? 0)
+    // A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
+    const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
+    const { rows: s } = await db.rawQuery(
+      `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
+       FROM releases ${sample} WHERE files > 0`
+    )
+    const sampled = Number(s[0].sampled)
+    const releases = sample ? Math.max(estimate, 0) : sampled
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    // History by articles; the pass: the history ranges before those deferred behind them (priority
+    // 2000 apart; e.g. groups added later that go first).
+    const { rows: ranges } = await db.rawQuery(
+      `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
+            pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
+       SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
+              coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
+              coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
+              coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
+              (SELECT max(done_at) FROM ranges) AS last_done_at,
+              (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
+       FROM h`
+    )
+    const g = ranges[0]
+    return {
+      approximate: sample !== '',
+      releases,
+      readableReleases: scale(s[0].readable),
+      files: scale(s[0].files),
+      size: scale(s[0].bytes),
+      ranges: { done: Number(g.done), total: Number(g.total) },
+      /** History and the current pass, in articles. */
+      history: { done: Number(g.history_done), total: Number(g.history) },
+      pass: { done: Number(g.pass_done), total: Number(g.pass) },
+      lastIndexedAt: g.last_done_at,
+      tailIndexedAt: g.tail_done_at,
+    }
+  }
+}
+
+export default new SearchService()

app/services/settler.ts

@@ -0,0 +1,47 @@
+import db from '@adonisjs/lucid/services/db'
+
+/** Releases unchanged for this long are settled (ranges are read in order; the merger adds within minutes). */
+const QUIET_HOURS = 6
+/**
+ * History: a release whose last post is QUIET_HOURS old gets nothing new from the tail, only from the
+ * neighbouring ranges' fragments, which the merger adds within minutes: settled once unchanged this long.
+ * Waiting QUIET_HOURS after it was read held every history release back from phase 2 for six hours after
+ * each reset (the hourly sweeps' resets kept the prober idle; LOG.md 2026-10-12 (13)).
+ */
+const QUIET_HISTORY_MINUTES = 30
+const BATCH = 50_000
+
+/**
+ * Settles releases (REBUILD.md, part 8): once nothing has been added for
+ * QUIET_HOURS (QUIET_HISTORY_MINUTES once its last post is that old), its completeness is decided: complete when every file the
+ * [i/N] counter announced was found, unknown without a counter (phase 2
+ * reads par2 sets for those). Phase 2 picks its work from settled releases.
+ */
+class Settler {
+  /** Up to `batches` batches of BATCH (all there are without). */
+  async run(batches = Infinity): Promise<number> {
+    let settled = 0
+    for (let batch = 0; batch < batches; batch++) {
+      // Their probe candidates become claimable (Prober.claim), the unreadable ones first.
+      const { rows } = await db.rawQuery(
+        `WITH settled AS (
+           UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
+           WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
+             AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
+           RETURNING id, readable, files, bytes, poster),
+         marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
+         -- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
+         queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
+           ON CONFLICT DO NOTHING)
+         SELECT count(*)::int AS n FROM settled`,
+        [QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
+      )
+      const n = rows[0].n
+      settled += n
+      if (n < BATCH) return settled
+    }
+    return settled
+  }
+}
+
+export default new Settler()

app/services/sidecar_folder.ts

@@ -0,0 +1,91 @@
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
+
+/** Candidates looked at per round. */
+const BATCH = 2000
+/** An upload's sidecar is posted with it: its release is looked for this far from it. */
+const WINDOW_MINUTES = 10
+/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
+const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
+
+/** The words of a name worth comparing: 4+ letters or digits. */
+function words(text: string): Set<string> {
+  return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
+}
+
+export interface Sibling {
+  id: number
+  name: string
+  files: number
+  file_total: number | null
+}
+
+/**
+ * Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
+ * ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
+ * under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
+ * isn't one release.
+ */
+export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
+  const said = words(files.map((f) => f.subject).join(' '))
+  const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
+  if (named.length === 1) return named[0]
+  if (named.length > 1) return null
+  const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
+  return counted.length === 1 ? counted[0] : null
+}
+
+export interface FoldRound {
+  looked: number
+  sidecars: number
+  folded: number
+}
+
+/**
+ * The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
+ * counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
+ * settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
+ * the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
+ * 'sidecar'). The rest leave the queue untouched.
+ */
+export class SidecarFolder {
+  constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}
+
+  async round(): Promise<FoldRound> {
+    const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
+    const { rows: queued } = await db.rawQuery(
+      `SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
+       FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
+      [BATCH]
+    )
+    const { rows: stale } = await db.rawQuery(
+      `SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
+      [BATCH]
+    )
+    const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
+    for (const c of queued) {
+      round.looked++
+      done.push(Number(c.id))
+      const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
+      const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
+      if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
+      round.sidecars++
+      if (!c.posted_at) continue
+      const { rows: siblings } = await db.rawQuery(
+        `SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
+         WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
+           AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
+        [c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
+      )
+      const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
+      if (!sibling) continue
+      const full = siblings.find((s: any) => Number(s.id) === sibling.id)
+      const merged = await db.transaction((trx) =>
+        this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
+      )
+      if (merged > 0) round.folded++
+    }
+    if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
+    return round
+  }
+}

app/services/spool.ts

@@ -0,0 +1,83 @@
+import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import coordinatorConfig from '#config/coordinator'
+
+export interface ChunkSpan {
+  from: number
+  to: number
+}
+
+/**
+ * Prefetched XOVER chunks on disk (REBUILD.md, prefetch): `<dir>/<range id>/<from>-<to>.gz`,
+ * each as the news server sent it. Chunks follow a fixed grid from the
+ * range's start, so a resume point (always at a batch boundary) is a chunk's start.
+ */
+class Spool {
+  get dir() {
+    return coordinatorConfig.spool.dir
+  }
+
+  /** The chunks of a range from `from` (its resume point) to its end. */
+  chunksOf(range: { start: number; end: number }, from = range.start): ChunkSpan[] {
+    const size = coordinatorConfig.chunkArticles
+    const spans: ChunkSpan[] = []
+    for (let at = range.start; at <= range.end; at += size) {
+      const span = { from: at, to: Math.min(at + size - 1, range.end) }
+      if (span.to >= from) spans.push(span)
+    }
+    return spans
+  }
+
+  private path(rangeId: number, span: ChunkSpan) {
+    return join(this.dir, String(rangeId), `${span.from}-${span.to}.gz`)
+  }
+
+  async has(rangeId: number, span: ChunkSpan) {
+    try {
+      await stat(this.path(rangeId, span))
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  async write(rangeId: number, span: ChunkSpan, data: Buffer) {
+    const file = this.path(rangeId, span)
+    await mkdir(join(this.dir, String(rangeId)), { recursive: true })
+    // Written under another name first: a reader never sees half a chunk.
+    await writeFile(`${file}.part`, data)
+    await rename(`${file}.part`, file)
+  }
+
+  read(rangeId: number, span: ChunkSpan) {
+    return readFile(this.path(rangeId, span))
+  }
+
+  /** One chunk, e.g. one that doesn't inflate (a false end of reply; see XoverClient.compressedRaw). */
+  async removeChunk(rangeId: number, span: ChunkSpan) {
+    await rm(this.path(rangeId, span), { force: true })
+  }
+
+  async remove(rangeId: number) {
+    await rm(join(this.dir, String(rangeId)), { recursive: true, force: true })
+  }
+
+  /** Bytes in the spool. */
+  async size(): Promise<number> {
+    let total = 0
+    let ranges: string[] = []
+    try {
+      ranges = await readdir(this.dir)
+    } catch {
+      return 0
+    }
+    for (const range of ranges) {
+      for (const file of await readdir(join(this.dir, range)).catch(() => [])) {
+        total += (await stat(join(this.dir, range, file)).catch(() => ({ size: 0 }))).size
+      }
+    }
+    return total
+  }
+}
+
+export default new Spool()

app/services/spot_inspector.ts

@@ -0,0 +1,359 @@
+import { createHash } from 'node:crypto'
+import { gzipSync } from 'node:zlib'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { XoverClient } from '#lib/nntp/index'
+import { decodeSpotBinary } from '#lib/spotnet/index'
+import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
+import { parseSubject } from '#lib/headers/index'
+import { releaseIdentity } from '#lib/release/index'
+
+/** Spots claimed per round, newest first. */
+const BATCH = 20
+/** Between rounds when there was nothing to do. */
+const IDLE_MS = 60_000
+/** Unmatched spots are matched again this often (the header pass may have built their releases since). */
+const REMATCH_MS = 60 * 60 * 1000
+/**
+ * Unmatched spots matched again per hour: after a reset every spot is (16.5k at 500 an hour took 33 h). Never tried
+ * first, then the newest: the oldest inspection first tried a 3-month backfill before today's spots, the ones whose
+ * releases the tail has (2 of 16.7k matched 3 h after reset 24).
+ */
+const REMATCH_BATCH = 5000
+/** Unmatched spots are matched again for this long after their inspection. */
+const REMATCH_DAYS = 30
+/** A file's release posted further apart than this from the NZB's date is another upload of the same name. */
+const MAX_SPREAD_MS = 48 * 3600 * 1000
+
+export interface InspectRound {
+  inspected: number
+  matched: number
+  standalone: number
+  outcomes: Record<string, number>
+}
+
+interface SpotFile {
+  idx: number
+  name: string
+  subject: string
+  poster: string
+  groups: string[]
+  postedAt: Date | null
+  bytes: number
+  segments: number
+}
+
+/**
+ * A placeholder NZB (Dummy.rar + Dummy.nzb, shared by many spots): under 1% of the size the spot announces, or under
+ * 16 KB in all (text-only spots). Its files aren't listed (voyager 1's isDummyNzb).
+ */
+export function isDummyNzb(nzb: ParsedNzb, spotSize: number): boolean {
+  const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
+  return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
+}
+
+/** A file of an NZB as the header pass names it: the subject's file name, by the same parser. */
+export function nzbFileName(subject: string): { name: string; base: string } {
+  const parsed = parseSubject(subject) ?? parseSubject(`${subject} (1/1)`)
+  return parsed ? { name: parsed.name, base: parsed.base } : { name: subject, base: subject }
+}
+
+/**
+ * The spot inspector (REBUILD.md, spots): looks inside each listed spot, on
+ * one NNTP connection of its own. It reads the spot's compressed NZB (its
+ * articles in alt.binaries.ftd), keeps the list of files in `spot_files`,
+ * and matches every file to the release the header pass built for it: the
+ * release identity of the file's subject and poster (lib/release, as the
+ * edge workers compute it), in one of the file's groups, posted within 48 h.
+ * Each matched release is linked in `release_spots`, and gets the spot's
+ * password in `release_passwords` (the NZB's own `<meta type="password">`
+ * first, else what the spot's title or text names) and `passworded`. A spot
+ * whose files match nothing is matched again every hour, for 30 days; once
+ * SPOT_STANDALONE_AFTER_HOURS have passed since its inspection it becomes a
+ * release of its own (source `spot`) with its NZB, until the header pass
+ * builds the releases after all (a spot older than what the catalog has
+ * indexed): then it moves to those, and its own release goes.
+ */
+export class SpotInspector {
+  private stopped = false
+  private client?: XoverClient
+  private groupIds = new Map<string, number | null>()
+  private rematchedAt = 0
+
+  stop() {
+    this.stopped = true
+    this.client?.close()
+  }
+
+  async run() {
+    while (!this.stopped) {
+      let round: InspectRound | null = null
+      try {
+        round = await this.round()
+        if (round.inspected > 0 || round.standalone > 0) {
+          const outcomes = Object.entries(round.outcomes).map(([k, n]) => `${n} ${k}`).join(', ')
+          logger.info(`Spots inspected: ${round.inspected} (${outcomes}); ${round.matched} matched releases, ${round.standalone} of their own.`)
+        }
+      } catch (error: any) {
+        logger.error(`Spot inspection failed: ${error.message}`)
+        this.client?.close()
+        this.client = undefined
+      }
+      if (!round || round.inspected === 0) {
+        for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+      }
+    }
+  }
+
+  private async connect(): Promise<XoverClient> {
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: false,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    return this.client
+  }
+
+  async round(): Promise<InspectRound> {
+    const round: InspectRound = { inspected: 0, matched: 0, standalone: 0, outcomes: {} }
+    const spots = await db.from('spots').where('status', 'listed').orderBy('posted_at', 'desc').limit(BATCH)
+    for (const spot of spots) {
+      if (this.stopped) break
+      const outcome = await this.inspect(spot)
+      round.inspected++
+      round.outcomes[outcome.status] = (round.outcomes[outcome.status] ?? 0) + 1
+      round.matched += outcome.matched
+    }
+    if (Date.now() - this.rematchedAt >= REMATCH_MS && !this.stopped) {
+      this.rematchedAt = Date.now()
+      const again = await this.rematch()
+      round.matched += again.matched
+      round.standalone += again.standalone
+    }
+    return round
+  }
+
+  /** One spot: its NZB read, its files listed and matched. */
+  async inspect(spot: any): Promise<{ status: string; matched: number }> {
+    const done = async (status: string, extra: Record<string, unknown> = {}) => {
+      await db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
+      return { status, matched: 0 }
+    }
+    const segments: string[] = spot.nzb_segments ?? []
+    if (segments.length === 0) return done('no-nzb')
+    if (segments.length > coordinatorConfig.spots.maxNzbSegments) return done('no-nzb', { error: `${segments.length} NZB articles` })
+    const client = await this.connect()
+    const bodies = await client.bodies(segments)
+    if (bodies.some((b) => b === null)) return done('nzb-gone', { error: `${bodies.filter((b) => b === null).length} of ${segments.length} NZB articles gone` })
+    let xml: string
+    try {
+      xml = decodeSpotBinary(bodies as Buffer[]).toString('utf8')
+    } catch (error: any) {
+      return done('nzb-unreadable', { error: String(error.message).slice(0, 500) })
+    }
+    return this.inspectNzb(spot, xml)
+  }
+
+  /** The files of a spot's NZB listed and matched (the tests feed this directly). */
+  async inspectNzb(spot: any, xml: string): Promise<{ status: string; matched: number }> {
+    const nzb = parseNzb(xml)
+    const finish = (status: string, extra: Record<string, unknown> = {}) =>
+      db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
+    if (!nzb || nzb.files.length === 0) {
+      await finish('no-nzb')
+      return { status: 'no-nzb', matched: 0 }
+    }
+    if (isDummyNzb(nzb, Number(spot.size ?? 0))) {
+      await finish('dummy')
+      return { status: 'dummy', matched: 0 }
+    }
+    const files: SpotFile[] = nzb.files.map((f, idx) => ({
+      idx,
+      name: nzbFileName(f.subject).name.slice(0, 1000),
+      subject: f.subject.slice(0, 2000),
+      poster: f.poster,
+      groups: f.groups,
+      postedAt: f.date ? new Date(f.date * 1000) : null,
+      bytes: f.segments.reduce((s, x) => s + x.bytes, 0),
+      segments: f.segments.length,
+    }))
+    // The NZB's own password over what the spot's text names.
+    const password = nzb.meta.password ? { password: nzb.meta.password.slice(0, 200), source: 'nzb' } : spot.password ? { password: spot.password, source: spot.password_source } : null
+    await db.transaction(async (trx) => {
+      await trx.from('spot_files').where('spot_id', spot.message_id).delete()
+      for (let i = 0; i < files.length; i += 1000) {
+        await trx.table('spot_files').multiInsert(
+          files.slice(i, i + 1000).map((f) => ({
+            spot_id: spot.message_id,
+            idx: f.idx,
+            name: f.name,
+            subject: f.subject,
+            poster: f.poster,
+            groups: f.groups,
+            posted_at: f.postedAt,
+            bytes: f.bytes,
+            segments: f.segments,
+          }))
+        )
+      }
+      await trx
+        .from('spots')
+        .where('message_id', spot.message_id)
+        .update({
+          status: 'inspected',
+          inspected_at: new Date(),
+          files: files.length,
+          matched: 0,
+          nzb: gzipSync(xml),
+          ...(password && password.source === 'nzb' ? { password: password.password, password_source: 'nzb' } : {}),
+        })
+    })
+    const matched = await this.match(spot.message_id)
+    return { status: 'inspected', matched }
+  }
+
+  private async groupId(name: string): Promise<number | null> {
+    if (!this.groupIds.has(name)) {
+      const row = await db.from('groups').where('name', name).select('id').first()
+      this.groupIds.set(name, row ? Number(row.id) : null)
+    }
+    return this.groupIds.get(name)!
+  }
+
+  /**
+   * Matches a spot's listed files to releases and links them; how many releases. A file's release: the same
+   * identity (releaseIdentity over its subject and poster) in one of its groups, the one posted nearest the file.
+   */
+  async match(spotId: string): Promise<number> {
+    const spot = await db.from('spots').where('message_id', spotId).first()
+    if (!spot || spot.status !== 'inspected') return 0
+    const files = await db.from('spot_files').where('spot_id', spotId).orderBy('idx')
+    const wanted: { idx: number; key: string; groups: number[]; at: number | null }[] = []
+    for (const f of files) {
+      if (!f.subject) continue
+      const { name, base } = nzbFileName(f.subject)
+      const identity = releaseIdentity({ poster: f.poster ?? '', subject: base, name, weak: false })
+      if (!identity) continue
+      const groups = (await Promise.all((f.groups ?? []).map((g: string) => this.groupId(g)))).filter((g): g is number => g !== null)
+      if (groups.length > 0) wanted.push({ idx: f.idx, key: identity.key, groups, at: f.posted_at ? new Date(f.posted_at).getTime() : null })
+    }
+    if (wanted.length === 0) return 0
+    const { rows } = await db.rawQuery(
+      `SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
+      [[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
+    )
+    const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
+    for (const r of rows) {
+      const list = byKey.get(r.identity_key) ?? []
+      list.push({ id: Number(r.id), group: Number(r.group_id), at: r.posted_at ? new Date(r.posted_at).getTime() : null })
+      byKey.set(r.identity_key, list)
+    }
+    const fileRelease = new Map<number, number>()
+    for (const w of wanted) {
+      const near = (byKey.get(w.key) ?? [])
+        .filter((r) => w.groups.includes(r.group))
+        .map((r) => ({ ...r, spread: r.at === null || w.at === null ? 0 : Math.abs(r.at - w.at) }))
+        .filter((r) => r.spread <= MAX_SPREAD_MS)
+        .sort((a, b) => a.spread - b.spread)
+      if (near[0]) fileRelease.set(w.idx, near[0].id)
+    }
+    const perRelease = new Map<number, number>()
+    for (const id of fileRelease.values()) perRelease.set(id, (perRelease.get(id) ?? 0) + 1)
+    if (perRelease.size === 0) return 0
+    await db.transaction(async (trx) => {
+      // A release of the spot's own goes now that the header pass built its files' releases.
+      if (spot.release_id) {
+        const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
+        if (own) {
+          await trx.from('release_spots').where('release_id', own.id).delete()
+          await trx.from('release_passwords').where('release_id', own.id).delete()
+          await trx.from('releases').where('id', own.id).delete()
+        }
+      }
+      for (const [idx, releaseId] of fileRelease) await trx.from('spot_files').where({ spot_id: spotId, idx }).update({ release_id: releaseId })
+      for (const [releaseId, n] of perRelease) {
+        await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: n }).onConflict(['release_id', 'spot_id']).merge({ files: n })
+      }
+      if (spot.password && perRelease.size > 0) {
+        await trx
+          .table('release_passwords')
+          .multiInsert([...perRelease.keys()].map((id) => ({ release_id: id, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId })))
+          .onConflict(['release_id', 'password'])
+          .ignore()
+        await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
+      }
+      const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
+      // The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
+      await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
+    })
+    return perRelease.size
+  }
+
+  /** Unmatched inspected spots matched again; the ones that waited long enough become releases of their own. */
+  async rematch(): Promise<{ matched: number; standalone: number }> {
+    const out = { matched: 0, standalone: 0 }
+    const { rows: spots } = await db.rawQuery(
+      `SELECT message_id, inspected_at, release_id FROM spots
+       WHERE status = 'inspected' AND matched = 0 AND inspected_at > now() - make_interval(days => ?)
+       ORDER BY rematched_at NULLS FIRST, posted_at DESC NULLS LAST LIMIT ?`,
+      [REMATCH_DAYS, REMATCH_BATCH]
+    )
+    const cutoff = Date.now() - coordinatorConfig.spots.standaloneAfterHours * 3600 * 1000
+    for (const spot of spots) {
+      if (this.stopped) break
+      const n = await this.match(spot.message_id)
+      await db.from('spots').where('message_id', spot.message_id).update({ rematched_at: new Date() })
+      out.matched += n
+      if (n === 0 && !spot.release_id && new Date(spot.inspected_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++
+    }
+    return out
+  }
+
+  /** A release of the spot's own (source `spot`), from its listed files; false when it has none. */
+  async standalone(spotId: string): Promise<boolean> {
+    const spot = await db.from('spots').where('message_id', spotId).first()
+    const files = await db.from('spot_files').where('spot_id', spotId)
+    if (!spot || files.length === 0) return false
+    const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
+    const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
+    const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
+    const [row] = await db
+      .table('releases')
+      .insert({
+        release_key: `spot:${key}`,
+        identity_key: key,
+        group_id: groupId,
+        name: spot.title.slice(0, 1000),
+        source: 'spot',
+        poster: spot.poster,
+        groups: JSON.stringify(groups),
+        posted_at: spot.posted_at,
+        last_posted_at: spot.posted_at,
+        files: files.length,
+        segments: files.reduce((s: number, f: any) => s + Number(f.segments), 0),
+        bytes: files.reduce((s: number, f: any) => s + Number(f.bytes), 0),
+        readable: true,
+        passworded: spot.password ? true : null,
+        settled_at: new Date(),
+      })
+      .onConflict('release_key')
+      .ignore()
+      .returning('id')
+    if (!row) return false
+    const releaseId = Number(row.id)
+    await db.transaction(async (trx) => {
+      await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
+      if (spot.password) {
+        await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
+      }
+      await trx.from('spots').where('message_id', spotId).update({ release_id: releaseId })
+    })
+    return true
+  }
+}

app/services/spotter.ts

@@ -0,0 +1,184 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import env from '#start/env'
+import coordinatorConfig from '#config/coordinator'
+import { MissingArticle, XoverClient } from '#lib/nntp/index'
+import {
+  decodeSpotText,
+  moderationTarget,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  passwordFromSpot,
+  spotterId,
+  verifySpotHeader,
+} from '#lib/spotnet/index'
+
+/** Overviews asked for at a time. */
+const XOVER_STEP = 1000
+/** A personal dispose counts within this long of the spot (Spotweb). */
+const DISPOSE_WINDOW_SECONDS = 5 * 86400
+
+export interface SpotterRound {
+  read: number
+  listed: number
+  unverified: number
+  moderation: number
+}
+
+/**
+ * The spotter (REBUILD.md, spots): Spotnet's spots, from the headers of
+ * free.pt into `spots`, on one NNTP connection of its own. Every few
+ * minutes it reads the overviews after the last article it saw (the first
+ * time the last SPOT_BACKFILL), and for each spot the full header: its
+ * signature is checked there (lib/spotnet, as Spotweb does), and the X-XML
+ * gives the description, the website and the NZB's articles. A password the
+ * title (`{{pw}}`) or the description names is kept with the spot; the spot
+ * inspector carries it to the releases. A delete/dispose message marks its
+ * target `removed` and takes back what the target gave the releases.
+ */
+export class Spotter {
+  private stopped = false
+  private client?: XoverClient
+
+  stop() {
+    this.stopped = true
+    this.client?.close()
+  }
+
+  async run() {
+    while (!this.stopped) {
+      const started = Date.now()
+      try {
+        const round = await this.round()
+        if (round.read > 0) {
+          logger.info(`Spots: ${round.read} articles read, ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
+        }
+      } catch (error: any) {
+        logger.error(`Spot round failed: ${error.message}`)
+        this.client?.close()
+        this.client = undefined
+      }
+      const wait = coordinatorConfig.spots.everySeconds * 1000
+      for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
+    }
+  }
+
+  private async connect(): Promise<XoverClient> {
+    this.client ??= await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT', 563),
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: false,
+      timeoutMs: coordinatorConfig.nntpTimeoutMs,
+    })
+    return this.client
+  }
+
+  /** The articles after the last one seen, up to the group's high. */
+  async round(): Promise<SpotterRound> {
+    const { group, backfill } = coordinatorConfig.spots
+    const client = await this.connect()
+    const { low, high } = await client.group(group)
+    const state = await db.from('spot_state').where('group_name', group).first()
+    let next = state ? Number(state.last_article) + 1 : Math.max(low, high - backfill + 1)
+    const round: SpotterRound = { read: 0, listed: 0, unverified: 0, moderation: 0 }
+    while (next <= high && !this.stopped) {
+      const to = Math.min(high, next + XOVER_STEP - 1)
+      const overviews = await client.xover(next, to)
+      round.read += overviews.length
+      for (const [number, fields] of overviews) {
+        if (this.stopped) return round
+        const messageId = (fields['message-id'] ?? '').replace(/^<|>$/g, '')
+        if (!messageId || !parseSpotHeader(fields.from ?? '', fields.subject ?? '', messageId, fields.date)) continue
+        const outcome = await this.read(client, messageId, number)
+        if (outcome) round[outcome]++
+      }
+      await db
+        .table('spot_state')
+        .insert({ group_name: group, last_article: to })
+        .onConflict('group_name')
+        .merge({ last_article: to })
+      next = to + 1
+    }
+    return round
+  }
+
+  /** One spot by its full header; what it was, or null when it was there already or is gone. */
+  async read(client: XoverClient, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
+    if (await db.from('spots').where('message_id', messageId).first()) return null
+    let raw: Buffer
+    try {
+      raw = await client.head(messageId)
+    } catch (error) {
+      if (error instanceof MissingArticle) return null
+      throw error
+    }
+    return this.store(raw.toString('latin1'), messageId, articleNumber)
+  }
+
+  /** Verifies and stores a spot from its raw header (the tests feed this directly). */
+  async store(raw: string, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
+    const { headers, xml: xmlText } = parseSpotArticleHeaders(raw)
+    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
+    if (!header) return null
+    const verified = verifySpotHeader(header)
+    const xml = verified ? parseSpotXml(decodeSpotText(xmlText)) : null
+    const target = verified ? moderationTarget(header) : null
+    const title = xml?.title || header.title
+    // Kept apart by where it came from: the title's {{pw}}, else the description's "Wachtwoord: …".
+    const fromTitle = verified && !target ? passwordFromSpot(title, null) : null
+    const fromText = verified && !target && !fromTitle ? passwordFromSpot('', xml?.description) : null
+    const status = !verified ? 'unverified' : target ? 'moderation' : 'listed'
+    await db
+      .table('spots')
+      .insert({
+        message_id: messageId,
+        article_number: articleNumber,
+        title: title.slice(0, 2000),
+        tag: header.tag || null,
+        poster: header.poster,
+        category: header.category,
+        subcats: JSON.stringify(header.subcats),
+        size: header.size || null,
+        posted_at: header.stamp ? new Date(header.stamp * 1000) : null,
+        key_id: header.keyId,
+        spotter_id: header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : null,
+        description: xml?.description || null,
+        website: xml?.website || null,
+        nzb_segments: xml?.nzbSegments.length ? xml.nzbSegments : null,
+        password: fromTitle ?? fromText,
+        password_source: fromTitle ? 'spot-title' : fromText ? 'spot-text' : null,
+        status,
+      })
+      .onConflict('message_id')
+      .ignore()
+    if (target) await this.moderate(header.selfSignedPubKey, header.stamp, target)
+    return status
+  }
+
+  /**
+   * A delete/dispose/remove: the target is `removed`, and its links and passwords go from the releases. As in
+   * Spotweb, a personal dispose (the poster's own key) counts only from the same spotter within 5 days; a
+   * moderator's (Spotnet's key) always.
+   */
+  private async moderate(pubKey: string, stamp: number, targetId: string) {
+    const target = await db.from('spots').where('message_id', targetId).first()
+    if (!target) return
+    const personal = pubKey.length > 50 ? spotterId(pubKey) : ''
+    if (personal) {
+      const age = stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
+      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
+    }
+    await db.transaction(async (trx) => {
+      await trx.from('spots').where('message_id', targetId).update({ status: 'removed' })
+      await trx.from('release_spots').where('spot_id', targetId).delete()
+      await trx.from('release_passwords').where('spot_id', targetId).delete()
+      // A release the spot made of its own goes with it.
+      if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', 'spot').delete()
+    })
+    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+  }
+}

app/validators/user.ts

@@ -14,6 +14,8 @@ export const signupValidator = vine.create({
   email: email().unique({ table: 'users', column: 'email' }),
   password: password(),
   passwordConfirmation: password().sameAs('password'),
+  // Needed when SIGNUP is `invite` (config/access.ts).
+  invite: vine.string().trim().maxLength(32).optional(),
 })
 
 /**

bin/console.ts

@@ -11,8 +11,21 @@
 |
 */
 
+/**
+ * Edge workers (edge:node) have no database or web server: what start/env
+ * requires for those gets a placeholder, so a node's .env needs only the
+ * coordinator's URL and its token.
+ */
+if (process.argv[2] === 'edge:node') {
+  const placeholders: Record<string, string> = {
+    NODE_ENV: 'production', PORT: '0', HOST: '127.0.0.1', LOG_LEVEL: 'info', APP_KEY: 'edge-node-has-no-app-key-0000000000', APP_URL: 'http://127.0.0.1',
+    SESSION_DRIVER: 'memory', DB_HOST: '127.0.0.1', DB_PORT: '5432', DB_USER: 'none', DB_DATABASE: 'none',
+  }
+  for (const [key, value] of Object.entries(placeholders)) process.env[key] ??= value
+}
+
 await import('reflect-metadata')
-const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
+const { Ignitor, prettyPrintError } = await import('@adonisjs/core/ignitor')
 
 /**
  * URL to the application root. AdonisJS need it to resolve

bin/server.ts

@@ -10,7 +10,7 @@
 */
 
 await import('reflect-metadata')
-const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
+const { Ignitor, prettyPrintError } = await import('@adonisjs/core/ignitor')
 
 /**
  * URL to the application root. AdonisJS need it to resolve

bin/test.ts

@@ -13,7 +13,7 @@
 process.env.NODE_ENV = 'test'
 
 import 'reflect-metadata'
-import { Ignitor, prettyPrintError } from '@adonisjs/core'
+import { Ignitor, prettyPrintError } from '@adonisjs/core/ignitor'
 import { configure, processCLIArgs, run } from '@japa/runner'
 
 /**

commands/BodyQueuePrune.ts

@@ -1,97 +0,0 @@
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import indexerConfig from '#config/indexer'
-import QueueService from '#services/QueueService'
-import RedisService from '#services/RedisService'
-import DeferredService, { isDeferred } from '#services/DeferredService'
-
-const BATCH = 2000
-
-/** Rebuilds the wait list (KEYS[1]) without the ids in the set KEYS[2], in one atomic pass. */
-const FILTER_LIST = `
-local ids = redis.call('LRANGE', KEYS[1], 0, -1)
-local keep = {}
-for _, id in ipairs(ids) do
-  if redis.call('SISMEMBER', KEYS[2], id) == 0 then keep[#keep + 1] = id end
-end
-redis.call('DEL', KEYS[1])
-for i = 1, #keep, 5000 do
-  redis.call('RPUSH', KEYS[1], unpack(keep, i, math.min(i + 4999, #keep)))
-end
-return #ids - #keep`
-
-/**
- * Removes waiting body jobs whose message-id is on the skip or defer list,
- * e.g. after adding a pattern (CONCEPT.md D24, D28). Deferred ones are
- * recorded in deferred_files first, so they can still be processed later.
- */
-export default class BodyQueuePrune extends BaseCommand {
-  public static commandName = 'body-queue:prune'
-  public static description = 'Drops waiting body jobs that are now skipped or deferred.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.boolean({ description: 'Only count' })
-  declare dryRun: boolean
-
-  @flags.boolean({ description: 'Drop deferred jobs without recording them (already recorded by an earlier run)' })
-  declare skipRecord: boolean
-
-  public async run() {
-    const redis = RedisService.client
-    const queue = QueueService.bodyQueue
-    const waitKey = queue.toKey('wait')
-    let skipped = 0
-    let deferred = 0
-    let scanned = 0
-    const dropped: string[] = []
-    try {
-      const ids: string[] = await redis.lrange(waitKey, 0, -1)
-      for (let i = 0; i < ids.length; i += BATCH) {
-        const batch = ids.slice(i, i + BATCH)
-        const data: [Error | null, string | null][] = await batch
-          .reduce((pipeline, id) => pipeline.hget(queue.toKey(id), 'data'), redis.pipeline())
-          .exec()
-        const drop: string[] = []
-        const toDefer = new Map<string, { articleNumber: number; header: Record<string, string> }[]>()
-        batch.forEach((id, j) => {
-          const job = data[j][1] ? JSON.parse(data[j][1]!) : null
-          if (!job?.header) return
-          const messageId = job.header['message-id'] ?? ''
-          if (isDeferred(job.header)) {
-            const list = toDefer.get(job.group) ?? []
-            list.push({ articleNumber: Number(job.articleNumber), header: job.header })
-            toDefer.set(job.group, list)
-            drop.push(id)
-            deferred++
-          } else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(messageId))) {
-            drop.push(id)
-            skipped++
-          }
-        })
-        scanned += batch.length
-        if (this.dryRun || drop.length === 0) continue
-        if (!this.skipRecord) for (const [group, articles] of toDefer) await DeferredService.record(group, articles)
-        dropped.push(...drop)
-      }
-      if (dropped.length > 0) {
-        const dropKey = queue.toKey('prune-ids')
-        await redis.del(dropKey)
-        for (let i = 0; i < dropped.length; i += BATCH) await redis.sadd(dropKey, ...dropped.slice(i, i + BATCH))
-        await redis.eval(FILTER_LIST, 2, waitKey, dropKey)
-        await redis.del(dropKey)
-        for (let i = 0; i < dropped.length; i += BATCH) {
-          await redis.unlink(...dropped.slice(i, i + BATCH).map((id) => queue.toKey(id)))
-        }
-      }
-      this.logger.info(
-        `Scanned ${scanned} waiting body jobs: ${this.dryRun ? 'would drop' : 'dropped'} ${deferred} deferred and ${skipped} skipped.`
-      )
-    } finally {
-      await QueueService.closeAll()
-      await RedisService.close()
-    }
-  }
-}

commands/ContentBackfill.ts

@@ -1,68 +0,0 @@
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import db from '@adonisjs/lucid/services/db'
-import QueueService from '#services/QueueService'
-import ContentService from '#services/ContentService'
-
-const BATCH = 1000
-
-/**
- * Queues completed files for content inspection (names and groupings from
- * par2 and RAR headers, CONCEPT.md D21) that were completed before it
- * existed. Costs one article download per file, done by worker:collection.
- */
-export default class ContentBackfill extends BaseCommand {
-  public static commandName = 'content:backfill'
-  public static description = 'Queues existing files with random names for content inspection.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.number({ description: 'Queue at most this many files' })
-  declare limit?: number
-
-  @flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
-  declare all: boolean
-
-  @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
-  declare archiveNzbs: boolean
-
-  public async run() {
-    let lastId = 0
-    let queued = 0
-    try {
-      while (!this.limit || queued < this.limit) {
-        // Files never inspected: no fingerprint on their binary and no archive listing.
-        const query = db
-          .from('files as f')
-          .join('binaries as b', 'b.file_id', 'f.id')
-          .where('f.id', '>', lastId)
-        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
-        else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
-        const files = await query
-          .orderBy('f.id')
-          .limit(BATCH)
-          .select('f.id', 'f.filename')
-        if (files.length === 0) break
-        lastId = Number(files.at(-1).id)
-
-        const jobs = files
-          .filter((f) => ContentService.isCandidate(f.filename))
-          .slice(0, this.limit ? this.limit - queued : undefined)
-          .map((f) => ({
-            name: 'process-collection',
-            data: { fileId: Number(f.id) },
-            // A new id per run: BullMQ ignores an add() for an id it still knows.
-            // Targeted backfills go ahead of whatever is waiting.
-            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
-          }))
-        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
-        queued += jobs.length
-      }
-      this.logger.info(`Queued ${queued} files for content inspection.`)
-    } finally {
-      await QueueService.closeAll()
-    }
-  }
-}

commands/DeferredExpandRanges.ts

@@ -1,32 +0,0 @@
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import NntpService from '#services/NntpService'
-import QueueService from '#services/QueueService'
-import DeferredService from '#services/DeferredService'
-
-/**
- * Sends recorded deferred ranges (fully obfuscated ngPost articles, CONCEPT.md
- * D28) to the body queue, oldest first. Every article in them is downloaded:
- * their headers can't be grouped into files.
- */
-export default class DeferredExpandRanges extends BaseCommand {
-  public static commandName = 'deferred:expand-ranges'
-  public static description = 'Queues the bodies of deferred article ranges (costly: every article is downloaded).'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.number({ description: 'Ranges to expand', default: 10 })
-  declare limit: number
-
-  public async run() {
-    try {
-      const { ranges, articles } = await DeferredService.expandRanges(this.limit)
-      this.logger.info(`Expanded ${ranges} ranges: ${articles} articles sent to the body queue.`)
-    } finally {
-      await NntpService.shutdown()
-      await QueueService.closeAll()
-    }
-  }
-}

commands/GcScheduler.ts

@@ -1,282 +0,0 @@
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import type { Queue } from 'bullmq'
-import db from '@adonisjs/lucid/services/db'
-import gcConfig from '#config/gc'
-import QueueService from '#services/QueueService'
-import RedisService from '#services/RedisService'
-import Shutdown from '#services/Shutdown'
-
-const CLEAN_BATCH = 1000
-const SCAN_COUNT = 1000
-
-export default class GcScheduler extends BaseCommand {
-  public static commandName = 'gc:scheduler'
-  public static description = 'Periodically removes old BullMQ jobs and stale binaries.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.boolean({ description: 'Run one sweep and exit' })
-  declare once: boolean
-
-  @flags.boolean({ description: 'Report what would be removed without changing anything' })
-  declare dryRun: boolean
-
-  public async run() {
-    this.logger.info(
-      `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
-    )
-
-    try {
-      do {
-        try {
-          await this.sweep()
-        } catch (err: any) {
-          this.logger.error(`GC sweep failed: ${err.message}`)
-        }
-        if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
-      } while (!this.once && !Shutdown.requested)
-    } finally {
-      await QueueService.closeAll()
-      await RedisService.close()
-    }
-  }
-
-  private async sweep() {
-    const { completedAgeSeconds, failedAgeSeconds } = gcConfig.jobs
-    for (const queue of [...QueueService.all, ...QueueService.legacy]) {
-      const completed = await this.cleanJobs(queue, 'completed', completedAgeSeconds)
-      const failed = await this.cleanJobs(queue, 'failed', failedAgeSeconds)
-      if (completed || failed) {
-        this.logger.info(
-          `${queue.name}: ${this.verb('remove', 'removed')} ${completed} completed and ${failed} failed jobs.`
-        )
-      }
-    }
-
-    const trimmed = await this.trimBodyQueue()
-    if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)
-
-    const binaries = await this.deleteStaleBinaries()
-    this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
-
-    if (!this.dryRun) {
-      const budget = Date.now() + gcConfig.deleteBudgetMs
-      const parts = await this.dropCopiedParts(budget)
-      const single = await this.deleteSingleArticleBinaries(budget)
-      const releases = await this.deleteEmptyReleases(budget)
-      if (parts || single || releases) {
-        this.logger.info(
-          `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.`
-        )
-      }
-    }
-
-    const deferred = await this.deleteOldDeferredFiles()
-    if (deferred) this.logger.info(`Deferred files: ${this.verb('delete', 'deleted')} ${deferred} past retention.`)
-
-    const { expired, scanned } = await this.expireLegacyPendingFiles()
-    if (scanned > 0) {
-      this.logger.info(
-        `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
-      )
-    }
-  }
-
-  /**
-   * Drops the oldest waiting body jobs above gcConfig.bodyQueueMax. BullMQ
-   * workers take jobs from the right end of the wait list, so the oldest are
-   * there.
-   */
-  private async trimBodyQueue(): Promise<number> {
-    const redis = RedisService.client
-    const key = QueueService.bodyQueue.toKey('wait')
-    const excess = (await redis.llen(key)) - gcConfig.bodyQueueMax
-    if (excess <= 0 || this.dryRun) return Math.max(excess, 0)
-    let dropped = 0
-    while (dropped < excess) {
-      const ids: string[] = (await redis.rpop(key, Math.min(5000, excess - dropped))) ?? []
-      if (ids.length === 0) break
-      await redis.unlink(...ids.map((id) => QueueService.bodyQueue.toKey(id)))
-      dropped += ids.length
-    }
-    return dropped
-  }
-
-  /**
-   * Deletes binaries (and, by cascade, their parts) that received no segment
-   * for `staleAfterSeconds`: incomplete ones are given up on; completed ones
-   * already live on in `files`; conflicted ones will never become a file.
-   * Completed ones still waiting for worker:file are kept.
-   */
-  private async deleteStaleBinaries(): Promise<number> {
-    const query = db
-      .from('binaries')
-      .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
-      .where((q) =>
-        q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
-      )
-
-    if (this.dryRun) {
-      const [{ count }] = await query.count('* as count')
-      return Number(count)
-    }
-    return Number(await query.delete())
-  }
-
-  private async deleteOldDeferredFiles(): Promise<number> {
-    const cutoff = new Date(Date.now() - gcConfig.deferredRetentionDays * 24 * 60 * 60 * 1000)
-    let total = 0
-    for (const [table, column] of [['deferred_files', 'first_seen_at'], ['deferred_ranges', 'created_at']]) {
-      const query = db.from(table).where(column, '<', cutoff)
-      if (this.dryRun) {
-        const [{ count }] = await query.count('* as count')
-        total += Number(count)
-      } else {
-        total += Number(await query.delete())
-      }
-    }
-    return total
-  }
-
-  // Keyset cursors over the tables, kept between sweeps so one pass over
-  // millions of rows is spread over several sweeps.
-  private singleCursor = 0
-  private releaseCursor = 0
-
-  /** Parts of header-derived binaries that are a file already (worker:file now drops them itself). */
-  private async dropCopiedParts(budget: number): Promise<number> {
-    let total = 0
-    while (Date.now() < budget) {
-      const result = await db.rawQuery(
-        `DELETE FROM parts WHERE binary_id IN (
-           SELECT b.id FROM binaries b
-           WHERE b.file_id IS NOT NULL AND b.size IS NULL
-             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id)
-           LIMIT ?)`,
-        [500]
-      )
-      total += result.rowCount ?? 0
-      if (!result.rowCount) break
-    }
-    return total
-  }
-
-  /** Header-derived binaries that never got a second article (gcConfig.binaries.singleArticleAfterSeconds). */
-  private async deleteSingleArticleBinaries(budget: number): Promise<number> {
-    const cutoff = new Date(Date.now() - gcConfig.binaries.singleArticleAfterSeconds * 1000)
-    let total = 0
-    while (Date.now() < budget) {
-      const { rows } = await db.rawQuery(
-        `WITH batch AS (
-           SELECT id FROM binaries WHERE id > ? ORDER BY id LIMIT ?
-         ), gone AS (
-           DELETE FROM binaries b USING batch
-           WHERE b.id = batch.id AND b.completed_at IS NULL AND b.size IS NULL AND b.total_parts > 1
-             AND b.last_seen_at < ? AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) <= 1
-           RETURNING 1)
-         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
-        [this.singleCursor, gcConfig.deleteBatch, cutoff]
-      )
-      total += Number(rows[0].deleted)
-      if (rows[0].last === null) {
-        this.singleCursor = 0
-        break
-      }
-      this.singleCursor = Number(rows[0].last)
-    }
-    return total
-  }
-
-  /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
-  private async deleteEmptyReleases(budget: number): Promise<number> {
-    const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)
-    let total = 0
-    while (Date.now() < budget) {
-      const { rows } = await db.rawQuery(
-        `WITH batch AS (
-           SELECT id FROM releases WHERE id > ? ORDER BY id LIMIT ?
-         ), gone AS (
-           DELETE FROM releases r USING batch
-           WHERE r.id = batch.id AND r.file_count = 0 AND r.first_seen_at < ?
-             AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
-             AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)
-             AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)
-             AND NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)
-           RETURNING 1)
-         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
-        [this.releaseCursor, gcConfig.deleteBatch, cutoff]
-      )
-      total += Number(rows[0].deleted)
-      if (rows[0].last === null) {
-        this.releaseCursor = 0
-        break
-      }
-      this.releaseCursor = Number(rows[0].last)
-    }
-    return total
-  }
-
-  /** Removes jobs in `state` that finished more than `ageSeconds` ago. */
-  private async cleanJobs(queue: Queue, state: 'completed' | 'failed', ageSeconds: number) {
-    const graceMs = ageSeconds * 1000
-    if (this.dryRun) {
-      const cutoff = Date.now() - graceMs
-      // Finished-job sets are sorted by finishedOn, so a score range counts them cheaply.
-      return RedisService.client.zcount(queue.toKey(state), '-inf', cutoff)
-    }
-
-    let total = 0
-    let removed: string[]
-    do {
-      removed = await queue.clean(graceMs, CLEAN_BATCH, state)
-      total += removed.length
-    } while (removed.length === CLEAN_BATCH)
-    return total
-  }
-
-  /**
-   * Pending-file hashes of the old header worker (and the pre-AdonisJS app)
-   * never expire. Give them a TTL rather than deleting them outright.
-   */
-  private async expireLegacyPendingFiles() {
-    const redis = RedisService.client
-    const { prefix, ttlSeconds } = gcConfig.legacyPendingFiles
-    let cursor = '0'
-    let scanned = 0
-    let expired = 0
-
-    do {
-      const [next, keys]: [string, string[]] = await redis.scan(
-        cursor,
-        'MATCH',
-        `${prefix}*`,
-        'COUNT',
-        SCAN_COUNT
-      )
-      cursor = next
-      if (keys.length === 0) continue
-      scanned += keys.length
-
-      const ttls: [Error | null, number][] = await keys
-        .reduce((pipeline, key) => pipeline.ttl(key), redis.pipeline())
-        .exec()
-      const orphaned = keys.filter((_, i) => ttls[i][1] === -1)
-      expired += orphaned.length
-
-      if (!this.dryRun && orphaned.length > 0) {
-        await orphaned
-          .reduce((pipeline, key) => pipeline.expire(key, ttlSeconds, 'NX'), redis.pipeline())
-          .exec()
-      }
-    } while (cursor !== '0')
-
-    return { scanned, expired }
-  }
-
-  private verb(present: string, past: string) {
-    return this.dryRun ? `would ${present}` : past
-  }
-}

commands/GroupAdd.ts

@@ -1,72 +0,0 @@
-import { BaseCommand, args, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import Group from '#models/group'
-import NntpService from '#services/NntpService'
-import { parseGroupResponse } from '#lib/nntp/group'
-
-/**
- * Adds newsgroups to index (or reactivates them). The scheduler picks them
- * up on its next tick, starting INDEX_INITIAL_BACKFILL articles back.
- */
-export default class GroupAdd extends BaseCommand {
-  public static commandName = 'group:add'
-  public static description = 'Adds newsgroups to index.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @args.spread({ description: 'Group names, e.g. alt.binaries.hdtv' })
-  declare names: string[]
-
-  @flags.boolean({ description: 'Deactivate the groups instead' })
-  declare disable: boolean
-
-  @flags.boolean({
-    description: 'Also download bodies to identify obfuscated posts (one body per article; bandwidth-heavy)',
-  })
-  declare bodies: boolean
-
-  @flags.number({ description: 'Start this many articles back instead of INDEX_INITIAL_BACKFILL (new groups only)' })
-  declare backfill?: number
-
-  @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
-  declare reset: boolean
-
-  public async run() {
-    for (const name of this.names) {
-      if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
-        this.logger.error(`Not a valid group name: ${name}`)
-        continue
-      }
-      const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
-      group.active = !this.disable
-      if (this.bodies) group.fetchBodies = true
-      if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
-        // The scheduler continues after last_queued_id.
-        const last = await this.lastArticle(name)
-        group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
-        group.lastIndexedId = group.lastQueuedId
-      }
-      await group.save()
-      this.logger.info(
-        `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
-          (this.backfill !== undefined ? `, starting after article ${group.lastQueuedId}` : '')
-      )
-    }
-    if (this.backfill !== undefined) await NntpService.shutdown()
-  }
-
-  private async lastArticle(name: string): Promise<bigint> {
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      return parseGroupResponse((await conn.group(name)).resp).last
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-    }
-  }
-}

commands/GroupsTop.ts

@@ -1,176 +0,0 @@
-import { writeFile } from 'node:fs/promises'
-import { setTimeout as sleep } from 'node:timers/promises'
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import NntpService from '#services/NntpService'
-import indexerConfig from '#config/indexer'
-import { parseGroupResponse } from '#lib/nntp/group'
-import { parseSubject } from '#lib/subject/index'
-
-interface GroupRate {
-  name: string
-  perMinute: number
-  sampled?: number
-  normal?: number
-  obfuscated?: number
-  skipped?: number
-  avgArticleBytes?: number
-  avgHeaderBytes?: number
-}
-
-/**
- * Lists the busiest newsgroups (by articles posted per minute, from two
- * LIST ACTIVE snapshots) and optionally samples their newest headers to
- * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)).
- *
- *   node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt
- */
-export default class GroupsTop extends BaseCommand {
-  public static commandName = 'groups:top'
-  public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' })
-  declare pattern: string
-
-  @flags.number({ description: 'Minutes between the two snapshots', default: 5 })
-  declare minutes: number
-
-  @flags.number({ description: 'Number of groups to list', default: 1000 })
-  declare limit: number
-
-  @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 })
-  declare sample: number
-
-  @flags.string({ description: 'Write the list to this file (default: print only)' })
-  declare out?: string
-
-  public async run() {
-    const conn = await NntpService.acquire()
-    let failed = false
-    try {
-      this.logger.info(`Snapshot 1 of ${this.pattern}…`)
-      const first = await this.snapshot(conn)
-      this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`)
-      const started = Date.now()
-      await sleep(this.minutes * 60_000)
-      const second = await this.snapshot(conn)
-      const minutes = (Date.now() - started) / 60_000
-
-      const groups: GroupRate[] = []
-      for (const [name, high] of second) {
-        const before = first.get(name)
-        if (before === undefined || high <= before) continue
-        groups.push({ name, perMinute: Number(high - before) / minutes })
-      }
-      groups.sort((a, b) => b.perMinute - a.perMinute)
-      const top = groups.slice(0, this.limit)
-      this.logger.info(`${groups.length} active groups; listing ${top.length}.`)
-
-      if (this.sample > 0) {
-        for (const [i, group] of top.entries()) {
-          try {
-            Object.assign(group, await this.sampleGroup(conn, group.name))
-          } catch (error: any) {
-            this.logger.warning(`${group.name}: ${error.message}`)
-          }
-          if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`)
-        }
-      }
-
-      const text = this.format(top, minutes)
-      if (this.out) {
-        await writeFile(this.out, text)
-        this.logger.info(`Written to ${this.out}`)
-      } else {
-        console.log(text)
-      }
-    } catch (error) {
-      failed = true
-      throw error
-    } finally {
-      NntpService.release(conn, failed)
-      await NntpService.shutdown()
-    }
-  }
-
-  /** Group name → highest article number, from LIST ACTIVE. */
-  private async snapshot(conn: any): Promise<Map<string, bigint>> {
-    const { data } = await conn.list(this.pattern)
-    const result = new Map<string, bigint>()
-    for (const line of Buffer.from(data).toString('latin1').split('\r\n')) {
-      const [name, high] = line.split(' ')
-      if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high))
-    }
-    return result
-  }
-
-  /** Classifies the newest headers the way worker:fetch does. */
-  private async sampleGroup(conn: any, name: string) {
-    const { last } = parseGroupResponse((await conn.group(name)).resp)
-    const start = last - BigInt(this.sample) + 1n
-    const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString())
-    let normal = 0
-    let skipped = 0
-    let obfuscated = 0
-    let obfuscatedBytes = 0
-    let headerBytes = 0
-    for (const [, header] of overviews as [number, Record<string, string>][]) {
-      if (!header) continue
-      headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0)
-      if (header.subject && parseSubject(header.subject)) normal++
-      else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
-      else {
-        obfuscated++
-        obfuscatedBytes += Number(header[':bytes']) || 0
-      }
-    }
-    const sampled = normal + skipped + obfuscated
-    return {
-      sampled,
-      normal,
-      skipped,
-      obfuscated,
-      avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0,
-      avgHeaderBytes: sampled ? headerBytes / sampled : 0,
-    }
-  }
-
-  private format(groups: GroupRate[], minutes: number): string {
-    const sampled = groups.some((g) => g.sampled)
-    const lines = [
-      `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`,
-      sampled
-        ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s'
-        : '# rank\tgroup\tarticles/min',
-    ]
-    let bodyTotal = 0
-    let headerTotal = 0
-    let articlesTotal = 0
-    for (const [i, g] of groups.entries()) {
-      articlesTotal += g.perMinute
-      if (!g.sampled) {
-        lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`)
-        continue
-      }
-      const obfuscatedShare = g.obfuscated! / g.sampled
-      // Bodies: every obfuscated article is downloaded once. Headers: every article.
-      const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6
-      const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6
-      bodyTotal += bodyMbit
-      headerTotal += headerMbit
-      lines.push(
-        [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t')
-      )
-    }
-    lines.push(`# total: ${Math.round(articlesTotal)} articles/min`)
-    if (sampled) {
-      lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`)
-      lines.push(`#   of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`)
-    }
-    return lines.join('\n') + '\n'
-  }
-}

commands/IndexScheduler.ts

@@ -1,119 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import Group from '#models/group'
-import NntpService from '#services/NntpService'
-import QueueService from '#services/QueueService'
-import indexerConfig from '#config/indexer'
-import Shutdown from '#services/Shutdown'
-import FetchRangeService from '#services/FetchRangeService'
-import SplitService from '#services/SplitService'
-
-export default class IndexScheduler extends BaseCommand {
-  public static commandName = 'index:scheduler'
-  public static description = 'Periodically checks for new articles and schedules them for fetching.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  private pool = NntpService
-
-  public async run() {
-    this.logger.info('Scheduler started. Awaiting tasks...')
-
-    const schedule = async () => {
-      this.logger.info('Checking for new headers...')
-      const groups = await Group.query().where('active', true)
-
-      if (groups.length === 0) {
-        this.logger.info('No active groups to index. Add some via `node ace db:seed` or manually.')
-        return
-      }
-
-      let conn
-      let failed = false
-      try {
-        conn = await this.pool.acquire()
-
-        for (const group of groups) {
-          try {
-            const { first: firstArticle, last: lastArticle } = await this.pool.selectGroup(
-              conn,
-              group.name
-            )
-
-            await FetchRangeService.requeue(group)
-
-            // Postgres returns bigint columns as strings.
-            const lastQueued = BigInt(group.lastQueuedId ?? 0)
-            const { batchSize, maxBatchesPerTick, initialBackfill } = indexerConfig
-            let startId =
-              lastQueued === 0n ? lastArticle - initialBackfill + 1n : lastQueued + 1n
-            if (startId < firstArticle) startId = firstArticle
-
-            if (startId > lastArticle) {
-              await FetchRangeService.advanceWatermark(group)
-              this.logger.info(`No new headers for group ${group.name}.`)
-              continue
-            }
-
-            let batches = 0
-            let endId = startId - 1n
-            while (endId < lastArticle && batches < maxBatchesPerTick) {
-              const batchStart = endId + 1n
-              endId = batchStart + batchSize - 1n < lastArticle ? batchStart + batchSize - 1n : lastArticle
-              await FetchRangeService.queue(group, batchStart, endId)
-              batches++
-            }
-
-            this.logger.info(
-              `Queued ${batches} fetch job(s) for ${group.name}: articles ${startId} to ${endId}` +
-                (endId < lastArticle ? ` (${lastArticle - endId} left for next tick)` : '')
-            )
-
-            group.lastQueuedId = endId
-            await group.save()
-            const watermark = await FetchRangeService.advanceWatermark(group)
-            if (endId - watermark > batchSize * BigInt(maxBatchesPerTick) * 3n) {
-              this.logger.warning(
-                `${group.name}: indexed up to ${watermark}, ${endId - watermark} articles behind the queue.`
-              )
-            }
-          } catch (err: any) {
-            failed = true
-            this.logger.error(`Error processing group ${group.name}: ${err.message}`)
-          }
-        }
-      } catch (err: any) {
-        failed = true
-        this.logger.error(`Error in scheduler main loop: ${err.message}`)
-      } finally {
-        if (conn) {
-          this.pool.release(conn, failed)
-        }
-      }
-    }
-
-    const split = async () => {
-      try {
-        const result = await SplitService.splitPending()
-        if (result.parents > 0) {
-          this.logger.info(
-            `Split ${result.parents} conflicted binaries: ${result.volumes} verified volumes, ${result.completed} completed.`
-          )
-        }
-      } catch (err: any) {
-        this.logger.error(`Error splitting conflicted binaries: ${err.message}`)
-      }
-    }
-
-    // Run immediately, then wait a full interval after each run so runs never overlap.
-    while (!Shutdown.requested) {
-      await schedule()
-      await split()
-      await Shutdown.sleep(indexerConfig.intervalMs)
-    }
-    await this.pool.shutdown()
-    await QueueService.closeAll()
-  }
-}

commands/ReleaseRebuild.ts

@@ -1,163 +0,0 @@
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import db from '@adonisjs/lucid/services/db'
-import IngestService from '#services/IngestService'
-import NameService from '#services/NameService'
-import QueueService from '#services/QueueService'
-import { rarReleaseIdentity, releaseIdentity } from '#lib/release/index'
-
-const BATCH = 5000
-
-/**
- * Recomputes the release of every binary (and its file) with the current
- * rules in lib/release. Used after changing those rules.
- */
-export default class ReleaseRebuild extends BaseCommand {
-  public static commandName = 'release:rebuild'
-  public static description = 'Reassigns all binaries and files to releases using the current grouping rules.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.boolean({
-    description:
-      'Also drop everything derived from file contents (par2 lists, par2/RAR groupings, real names), ' +
-      'to rebuild it with content:backfill --all',
-  })
-  declare resetContent: boolean
-
-  public async run() {
-    if (this.resetContent) await this.dropContentDerived()
-
-    let lastId = 0
-    let assigned = 0
-    let unassigned = 0
-    try {
-      while (true) {
-        const binaries = await db
-          .from('binaries')
-          .where('id', '>', lastId)
-          .orderBy('id')
-          .limit(BATCH)
-          .select('id', 'poster', 'subject', 'name', 'groups', 'posted_at')
-        if (binaries.length === 0) break
-        lastId = Number(binaries.at(-1).id)
-
-        const segments = binaries.map((binary) => {
-          // Body-derived binaries store a synthetic subject, see segmentFromYenc.
-          const weak = /^"[^"]*" yEnc$/.test(binary.subject)
-          return {
-            id: Number(binary.id),
-            poster: binary.poster,
-            groups: binary.groups,
-            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
-            weakIdentity: weak,
-            release: releaseIdentity({ poster: binary.poster, subject: binary.subject, name: binary.name, weak }),
-          }
-        })
-
-        await db.transaction(async (trx) => {
-          const releaseIds = await IngestService.upsertReleases(trx.knexClient, segments)
-          const ids = segments.map((s) => s.id)
-          const releaseIdsPerBinary = segments.map((s) =>
-            s.release ? releaseIds.get(s.release.key)! : null
-          )
-          await trx.rawQuery(
-            `UPDATE binaries b SET release_id = v.release_id
-             FROM unnest(?::bigint[], ?::bigint[]) AS v(id, release_id)
-             WHERE b.id = v.id AND b.release_id IS DISTINCT FROM v.release_id
-               AND NOT (v.release_id IS NULL
-                        AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))
-               -- A posted NZB decided this grouping (NzbImportService); name rules don't override it.
-               AND NOT EXISTS (SELECT 1 FROM files n WHERE n.release_id = b.release_id AND n.content_type = 'nzb')`,
-            [ids, releaseIdsPerBinary] as any
-          )
-        })
-        assigned += segments.filter((s) => s.release).length
-        unassigned += segments.filter((s) => !s.release).length
-      }
-
-      // Files grouped by their RAR headers: regroup from the stored listing (no downloads).
-      let rarGrouped = 0
-      let rarCursor = 0
-      while (true) {
-        const rows = await db
-          .from('files as f')
-          .join('binaries as b', 'b.file_id', 'f.id')
-          .whereNotNull('f.archive')
-          .whereNull('b.release_id')
-          .where('f.id', '>', rarCursor)
-          .orderBy('f.id')
-          .limit(BATCH)
-          .select('f.id as file_id', 'f.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
-        if (rows.length === 0) break
-        rarCursor = Number(rows.at(-1).file_id)
-        const segments = rows.map((r) => ({
-          fileId: Number(r.file_id),
-          binaryId: Number(r.binary_id),
-          poster: r.poster,
-          groups: r.groups,
-          postedAt: r.posted_at ? new Date(r.posted_at) : undefined,
-          weakIdentity: true,
-          release: rarReleaseIdentity(r.archive?.entries ?? []),
-        }))
-        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), segments)
-        for (const s of segments.filter((x) => x.release)) {
-          const releaseId = ids.get(s.release!.key)
-          await db.from('binaries').where('id', s.binaryId).update({ release_id: releaseId })
-          await db.from('files').where('id', s.fileId).update({ release_id: releaseId })
-          rarGrouped++
-        }
-      }
-
-      // Real names from par2 for binaries with a known size (no downloads).
-      let named = 0
-      let nameCursor = 0
-      while (true) {
-        const rows = await db.from('binaries').whereNotNull('size').where('id', '>', nameCursor).orderBy('id').limit(BATCH).select('id')
-        if (rows.length === 0) break
-        nameCursor = Number(rows.at(-1).id)
-        named += await NameService.resolve(rows.map((r) => Number(r.id)))
-      }
-
-      const files = await db.rawQuery(
-        `UPDATE files f SET release_id = b.release_id FROM binaries b
-         WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`
-      )
-      const orphans = await db.rawQuery(
-        `DELETE FROM releases r WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
-           AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)`
-      )
-      if (this.resetContent) {
-        this.logger.info('Content-derived data dropped; run `node ace content:backfill --all` to rebuild it.')
-      }
-      this.logger.info(
-        `Binaries: ${assigned} in a release, ${unassigned} without, ${rarGrouped} regrouped from RAR headers, ` +
-          `${named} named from par2. ` +
-          `Files updated: ${files.rowCount}. Empty releases removed: ${orphans.rowCount}.`
-      )
-    } finally {
-      await QueueService.closeAll()
-    }
-  }
-
-  /** par2 lists, par2/RAR release assignments and recovered names. */
-  private async dropContentDerived() {
-    await db.transaction(async (trx) => {
-      await trx.rawQuery('DELETE FROM release_files')
-      await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`)
-      await trx.rawQuery(`UPDATE files SET real_name = NULL, archive = NULL WHERE real_name IS NOT NULL OR archive IS NOT NULL`)
-      await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`)
-      // Binaries whose name rules find no release were grouped from contents (or wrongly by size).
-      await trx.rawQuery(`
-        UPDATE binaries SET release_id = NULL
-        WHERE subject ~ '^"[^"]*" yEnc$' AND release_id IS NOT NULL
-          AND (name !~ '\\.[A-Za-z0-9]{1,5}$' OR release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))`)
-      await trx.rawQuery(`
-        UPDATE files f SET release_id = b.release_id FROM binaries b
-        WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`)
-    })
-    this.logger.info('Dropped par2 lists, content-derived groupings and recovered names.')
-  }
-}

commands/ReleaseRename.ts

@@ -1,20 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import { renameFromStoredContents } from '#services/ArchiveService'
-import { regroupAndRenameFromPar2 } from '#services/NameService'
-
-/** Renames random-named releases from their stored archive listings (no downloads). */
-export default class ReleaseRename extends BaseCommand {
-  public static commandName = 'release:rename'
-  public static description = 'Renames random-named releases from their par2 listings and archive contents.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  public async run() {
-    const par2 = await regroupAndRenameFromPar2()
-    this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`)
-    this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`)
-  }
-}

commands/ReleaseStats.ts

@@ -1,57 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import db from '@adonisjs/lucid/services/db'
-
-/** Prints how well files are being grouped into releases. */
-export default class ReleaseStats extends BaseCommand {
-  public static commandName = 'release:stats'
-  public static description = 'Shows release recognition statistics.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  public async run() {
-    const { rows } = await db.rawQuery(`
-      WITH r AS (
-        SELECT r.id, r.file_total, r.poster IS NULL AS weak, r.source,
-               count(b.id) AS binaries,
-               count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed
-        FROM releases r LEFT JOIN binaries b ON b.release_id = r.id
-        GROUP BY r.id
-      )
-      SELECT
-        count(*) AS releases,
-        count(*) FILTER (WHERE completed > 0) AS with_files,
-        count(*) FILTER (WHERE completed >= 2) AS multi_file,
-        count(*) FILTER (WHERE file_total IS NOT NULL AND completed >= file_total) AS complete_by_counter,
-        count(*) FILTER (WHERE weak) AS from_obfuscated,
-        count(*) FILTER (WHERE weak AND completed > 0) AS from_obfuscated_with_files
-      FROM r`)
-    const { rows: coverage } = await db.rawQuery(`
-      SELECT count(*) AS completed_files,
-             count(*) FILTER (WHERE release_id IS NOT NULL) AS in_release
-      FROM binaries WHERE completed_at IS NOT NULL`)
-    const { rows: bySource } = await db.rawQuery(`
-      SELECT coalesce(r.source, 'unknown') AS source, count(DISTINCT r.id) AS releases,
-             count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed_files
-      FROM releases r JOIN binaries b ON b.release_id = r.id GROUP BY 1 ORDER BY 2 DESC`)
-    // Readable names: descriptive as posted, or recovered from contents.
-    const { rows: naming } = await db.rawQuery(`
-      WITH f AS (
-        SELECT real_name IS NOT NULL AS recovered,
-               is_readable_name(coalesce(real_name, filename)) AS readable_name,
-               regexp_replace(coalesce(real_name, filename),
-                 '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|mkv|mp4|avi|flac|mp3|jpg))+$', '', 'i') AS stem,
-               coalesce(real_name, filename) AS name
-        FROM files
-      )
-      SELECT count(*) AS files,
-             count(*) FILTER (WHERE recovered) AS names_recovered,
-             count(*) FILTER (WHERE name ~ '\\.[A-Za-z0-9]{1,5}$' AND readable_name) AS readable
-      FROM f`)
-    this.logger.info(JSON.stringify({ ...rows[0], ...coverage[0] }))
-    this.logger.info(`naming: ${JSON.stringify(naming[0])}`)
-    this.logger.info(`by source: ${JSON.stringify(bySource)}`)
-  }
-}

commands/access_invite.ts

@@ -0,0 +1,37 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import AccessService from '#services/access_service'
+import { TIERS } from '#config/access'
+
+/** Makes invite codes (SIGNUP=invite), optionally with a tier for the new account, e.g. a sold VIP year. */
+export default class AccessInvite extends BaseCommand {
+  static commandName = 'access:invite'
+  static description = 'Makes invite codes, optionally giving a tier (e.g. --tier=vip --days=365)'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'How many codes', default: 1 })
+  declare count: number
+
+  @flags.string({ description: `Tier the new account gets (${Object.keys(TIERS).join(', ')})` })
+  declare tier?: string
+
+  @flags.number({ description: 'For how many days the tier holds (without: no end)' })
+  declare days?: number
+
+  @flags.string({ description: 'A note, e.g. who it is for or the payment' })
+  declare note?: string
+
+  async run() {
+    if (this.tier && !TIERS[this.tier]) {
+      this.logger.error(`No such tier: ${this.tier}`)
+      this.exitCode = 1
+      return
+    }
+    const codes = Array.from({ length: this.count }, () => AccessService.newKey())
+    await db.table('invites').multiInsert(
+      codes.map((code) => ({ code, tier: this.tier ?? null, tier_days: this.tier ? (this.days ?? null) : null, note: this.note ?? null }))
+    )
+    for (const code of codes) this.logger.log(code)
+  }
+}

commands/access_suspend.ts

@@ -0,0 +1,28 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import User from '#models/user'
+
+/** Suspends a user (Newznab error 101, the catalog 403), e.g. for a shared key; --undo lifts it. */
+export default class AccessSuspend extends BaseCommand {
+  static commandName = 'access:suspend'
+  static description = 'Suspends a user (or lifts it with --undo)'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'The user’s email' })
+  declare email: string
+
+  @flags.boolean({ description: 'Lift the suspension' })
+  declare undo: boolean
+
+  async run() {
+    const user = await User.findBy('email', this.email)
+    if (!user) {
+      this.logger.error(`No user ${this.email}`)
+      this.exitCode = 1
+      return
+    }
+    user.suspended = !this.undo
+    await user.save()
+    this.logger.info(`${user.email}: ${user.suspended ? 'suspended' : 'active'}`)
+  }
+}

commands/access_tier.ts

@@ -0,0 +1,32 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import User from '#models/user'
+import AccessService from '#services/access_service'
+
+/** Gives a user a tier (a payment, by hand until one is taken on the site); a running one is extended. */
+export default class AccessTier extends BaseCommand {
+  static commandName = 'access:tier'
+  static description = "Sets a user's tier, e.g. access:tier someone@example.com vip --days=365"
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'The user’s email' })
+  declare email: string
+
+  @args.string({ description: 'Tier (config/access.ts)' })
+  declare tier: string
+
+  @flags.number({ description: 'For how many days (without: no end); the same tier still running is extended' })
+  declare days?: number
+
+  async run() {
+    const user = await User.findBy('email', this.email)
+    if (!user) {
+      this.logger.error(`No user ${this.email}`)
+      this.exitCode = 1
+      return
+    }
+    AccessService.grantTier(user, this.tier, this.days ?? null)
+    await user.save()
+    this.logger.info(`${user.email}: ${user.tier} ${user.tierUntil ? `until ${user.tierUntil.toISO()}` : 'without end'}`)
+  }
+}

commands/coordinator_run.ts

@@ -0,0 +1,136 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { Prefetcher } from '#services/prefetcher'
+import Planner from '#services/planner'
+import FragmentMerger from '#services/fragment_merger'
+import Settler from '#services/settler'
+import RangeBlocks from '#services/range_blocks'
+import NntpPool from '#services/nntp_pool'
+import coordinatorConfig from '#config/coordinator'
+
+const TICK_MS = 2000
+const PLAN_EVERY_MS = 60 * 60 * 1000
+const MERGE_EVERY_MS = 60 * 1000
+const MERGE_BUDGET_MS = 30 * 1000
+const STOP_WAIT_MS = 15 * 1000
+/**
+ * Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
+ * rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
+ * (LOG.md 2026-10-16 (67)).
+ */
+const SETTLE_EVERY_MS = 5 * 60 * 1000
+const SETTLE_BATCHES = 4
+/** range_blocks (release_final) after ranges finish. */
+const BLOCKS_EVERY_MS = 5 * 60 * 1000
+
+/**
+ * The coordinator's background work (REBUILD.md, part 1): the prefetcher,
+ * merging fragments of finished ranges every minute, and every hour
+ * planning ranges for new articles and settling quiet releases. The API (leases, stream,
+ * batches) runs in the web server.
+ */
+export default class CoordinatorRun extends BaseCommand {
+  static commandName = 'coordinator:run'
+  static description = 'Prefetches ranges into the spool and plans new ones'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.string({ description: 'This process\'s share of the prefetching, "i/n" (ranges with id % n = i - 1); only 1/n plans, merges and settles' })
+  declare shard?: string
+
+  async run() {
+    const [index, count] = (this.shard ?? '1/1').split('/').map(Number)
+    if (!(count >= 1 && index >= 1 && index <= count)) throw new Error(`--shard: "i/n" with 1 <= i <= n, not "${this.shard}"`)
+    const prefetcher = new Prefetcher({ index: index - 1, count })
+    const main = index === 1
+    let stopping = false
+    let stopped: () => void
+    const done = new Promise<void>((resolve) => (stopped = resolve))
+    // bin/console turns SIGTERM into app.terminate: wait for the spooling in flight first.
+    this.app.terminating(async () => {
+      this.logger.info('Stopping: waiting for the chunks in flight.')
+      stopping = true
+      prefetcher.stop()
+      await done
+    })
+    this.logger.info(count > 1 ? `Coordinator started (shard ${index}/${count}${main ? '' : ', prefetching only'}).` : 'Coordinator started.')
+    // Two loops: the prefetcher's ticks don't wait for merging or planning (a merge round
+    // of up to MERGE_BUDGET_MS left the connections idle once their targets were fetched).
+    const prefetching = (async () => {
+      while (!stopping) {
+        try {
+          await prefetcher.tick()
+        } catch (error: any) {
+          this.logger.error(`Prefetch round failed: ${error.message}`)
+        }
+        await sleep(TICK_MS)
+      }
+    })()
+    // range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
+    const blocks = (async () => {
+      while (main && !stopping) {
+        try {
+          await RangeBlocks.rebuild()
+        } catch (error: any) {
+          this.logger.error(`Range blocks failed: ${error.message}`)
+        }
+        for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
+      }
+    })()
+    // History for new groups hourly; the tail every few minutes.
+    let plannedAt = 0
+    let tailedAt = 0
+    let mergedAt = 0
+    let settledAt = 0
+    while (!stopping) {
+      try {
+        if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
+          plannedAt = Date.now()
+          const { groups, ranges } = await Planner.planAll()
+          if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
+        }
+        if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
+          settledAt = Date.now()
+          const settled = await Settler.run(SETTLE_BATCHES)
+          if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
+        }
+        if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
+          tailedAt = Date.now()
+          const { groups, ranges } = await Planner.planTail()
+          if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
+        }
+        if (main && Date.now() - mergedAt >= MERGE_EVERY_MS) {
+          mergedAt = Date.now()
+          // Rounds until a whole pass over the file keys decides nothing more, or MERGE_BUDGET_MS.
+          // (Going on while only waiting files were left kept Postgres busy all the time.)
+          const totals = { completed: 0, incomplete: 0, waiting: 0 }
+          let decidedThisPass = 0
+          while (!stopping && Date.now() - mergedAt < MERGE_BUDGET_MS) {
+            const merged = await FragmentMerger.run()
+            totals.completed += merged.completed
+            totals.incomplete += merged.incomplete
+            totals.waiting = merged.waiting
+            decidedThisPass += merged.completed + merged.incomplete
+            if (merged.wrapped) {
+              if (decidedThisPass === 0) break
+              decidedThisPass = 0
+            }
+          }
+          if (totals.completed || totals.incomplete) this.logger.info(`Fragments: ${totals.completed} files completed, ${totals.incomplete} incomplete, ${totals.waiting} waiting.`)
+        }
+      } catch (error: any) {
+        this.logger.error(`Coordinator round failed: ${error.message}`)
+      }
+      await sleep(TICK_MS)
+    }
+    await Promise.all([prefetching, blocks])
+    // The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
+    const waitUntil = Date.now() + STOP_WAIT_MS
+    while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
+    if (prefetcher.active > 0) this.logger.info(`Closing ${prefetcher.active} connections still fetching.`)
+    await NntpPool.close()
+    this.logger.info('Coordinator stopped.')
+    stopped!()
+    await this.terminate()
+  }
+}

commands/edge_bench.ts

@@ -0,0 +1,189 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { unzipSync } from 'node:zlib'
+import env from '#start/env'
+import { XoverClient } from '#lib/nntp/index'
+import { RangeIndexer } from '#lib/edge/index'
+import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
+import { decodeChunk, encodeChunk } from '#lib/chunks/index'
+import db from '@adonisjs/lucid/services/db'
+import BatchStore from '#services/batch_store'
+
+/**
+ * Measures the edge worker on a real range (REBUILD.md: TypeScript or Rust):
+ * --capture saves a range's compressed XOVER chunks to a directory once;
+ * without it the saved chunks are replayed through the range indexer, timing
+ * inflate, parse and indexing per chunk, with memory and output sizes.
+ */
+export default class EdgeBench extends BaseCommand {
+  static commandName = 'edge:bench'
+  static description = 'Measures the edge worker on a captured range'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.string({ description: 'Directory of the captured chunks', required: true })
+  declare dir: string
+
+  @flags.boolean({ description: 'Capture the range first (needs NNTP_* in the environment)' })
+  declare capture: boolean
+
+  @flags.string({ description: 'Group to capture' })
+  declare group?: string
+
+  @flags.number({ description: 'First article to capture' })
+  declare start?: number
+
+  @flags.number({ description: 'Articles to capture', default: 2_000_000 })
+  declare articles: number
+
+  @flags.number({ description: 'Articles per XOVER', default: 50_000 })
+  declare chunk: number
+
+  @flags.number({ description: 'Chunks per batch (results taken every N chunks)', default: 10 })
+  declare batchChunks: number
+
+  @flags.number({ description: 'Chunks without a new part before an open file is given up', default: 5 })
+  declare idle: number
+
+  @flags.boolean({ description: 'Decode every release chunk and encode it again: the same bytes' })
+  declare verify: boolean
+
+  @flags.boolean({ description: 'Also store the batches (BatchStore, the configured database), timed' })
+  declare store: boolean
+
+  @flags.string({ description: 'Indexed groups in priority order, comma-separated (owner rule)' })
+  declare priorities?: string
+
+  async run() {
+    if (this.capture) await this.captureRange()
+    await this.replay()
+  }
+
+  private async captureRange() {
+    if (!this.group || this.start === undefined) throw new Error('--group and --start are needed to capture')
+    await mkdir(this.dir, { recursive: true })
+    const client = await XoverClient.connect({
+      host: env.get('NNTP_HOST')!,
+      port: env.get('NNTP_PORT')!,
+      user: env.get('NNTP_USER'),
+      password: env.get('NNTP_PASSWORD'),
+      secure: env.get('NNTP_SECURE', true),
+      compress: true,
+      timeoutMs: 300_000,
+    })
+    await client.group(this.group)
+    const end = this.start + this.articles - 1
+    let index = 0
+    for (let from = this.start; from <= end; from += this.chunk, index++) {
+      const to = Math.min(from + this.chunk - 1, end)
+      const started = Date.now()
+      let raw = await client.xoverRaw(from, to)
+      // Eweka cuts some ranges' compressed reply short (the same bytes every time): fetched plain then.
+      try {
+        unzipSync(raw)
+      } catch {
+        const plain = await XoverClient.connect({ host: env.get('NNTP_HOST')!, port: env.get('NNTP_PORT')!, user: env.get('NNTP_USER'), password: env.get('NNTP_PASSWORD'), secure: env.get('NNTP_SECURE', true), compress: false, timeoutMs: 300_000 })
+        await plain.group(this.group)
+        raw = await plain.xoverRaw(from, to)
+        plain.close()
+        this.logger.warning(`chunk ${index}: the compressed reply didn't inflate; fetched plain`)
+      }
+      await writeFile(join(this.dir, `${String(index).padStart(4, '0')}-${from}-${to}.gz`), raw)
+      this.logger.info(`chunk ${index}: ${from}–${to}, ${(raw.length / 1e6).toFixed(1)} MB in ${((Date.now() - started) / 1000).toFixed(1)} s`)
+    }
+    await writeFile(join(this.dir, 'range.json'), JSON.stringify({ group: this.group, start: this.start, end }))
+    client.close()
+  }
+
+  private async replay() {
+    const { group, start, end } = JSON.parse(await readFile(join(this.dir, 'range.json'), 'utf8')) as { group: string; start: number; end: number }
+    const priorities = Object.fromEntries((this.priorities ?? group).split(',').map((g, i) => [g.trim(), i + 1]))
+    if (!(group in priorities)) priorities[group] = Object.keys(priorities).length + 1
+    const rules: Rules = { ...DEFAULT_RULES, priorities, droppedSignatures: [] }
+    const files = (await readdir(this.dir)).filter((f) => f.endsWith('.gz')).sort()
+    const indexer = new RangeIndexer({ group, start, end, rules, idleChunks: this.idle })
+    const t = { inflate: 0, parse: 0, index: 0 }
+    let compressed = 0
+    let text = 0
+    let peakRss = 0
+    let peakDetail = ''
+    const out = { headers: 0, releases: 0, files: 0, segments: 0, payload: 0, candidates: 0, fragments: 0, samples: 0 }
+    const counts = new Map<string, number>()
+    const keys = new Set<string>()
+    let batches = 0
+    let storeMs = 0
+    let rangeId = 0
+    let nodeId = 0
+    if (this.store) {
+      // Inactive: the coordinator neither plans nor leases it.
+      const [g] = await db.table('groups').insert({ name: `bench:${group}:${Date.now()}`, priority: 1_000_000 + (Date.now() % 1_000_000), active: false }).returning('id')
+      const [n] = await db.table('nodes').insert({ name: `bench-${Date.now()}` }).returning('id')
+      nodeId = Number(n.id)
+      const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: end, next_id: start, status: 'leased', node_id: nodeId }).returning('id')
+      rangeId = Number(r.id)
+    }
+    const cpuStart = process.cpuUsage()
+    const wall = Date.now()
+    for (const file of files) {
+      const raw = await readFile(join(this.dir, file))
+      compressed += raw.length
+      let at = performance.now()
+      const data = unzipSync(raw).toString('utf8')
+      t.inflate += performance.now() - at
+      text += data.length
+      at = performance.now()
+      const next = Number(file.split('-')[2].replace('.gz', '')) + 1
+      // The worker's path: the reply's text straight into the indexer (parsing is part of "index").
+      indexer.addText(data, next)
+      const last = file === files.at(-1)
+      if (last) indexer.finish()
+      t.index += performance.now() - at
+      if (!last && (files.indexOf(file) + 1) % this.batchChunks !== 0) continue
+      at = performance.now()
+      const batch = indexer.take()
+      t.index += performance.now() - at
+      if (this.store) {
+        const storedAt = performance.now()
+        await BatchStore.store(rangeId, nodeId, batch)
+        storeMs += performance.now() - storedAt
+      }
+      out.headers += batch.headers
+      out.releases += batch.releases.length
+      batches++
+      for (const r of batch.releases) {
+        if (this.verify) {
+          const decoded = decodeChunk(r.payload)
+          if (!encodeChunk(decoded).equals(r.payload) || decoded.reduce((n, f) => n + f.segments.length, 0) !== r.segments) {
+            throw new Error(`Release ${r.key}: its chunk doesn't decode to itself`)
+          }
+        }
+        keys.add(r.key)
+        out.files += r.files
+        out.segments += r.segments
+        out.payload += r.payload.length
+      }
+      out.candidates += batch.candidates.length
+      out.fragments += batch.fragments.length
+      out.samples += batch.samples.length
+      for (const c of batch.counts) counts.set(`${c.rule} ${c.key}`.trim(), (counts.get(`${c.rule} ${c.key}`.trim()) ?? 0) + c.articles)
+      const m = process.memoryUsage()
+      if (m.rss > peakRss) {
+        peakRss = m.rss
+        peakDetail = `heap ${(m.heapUsed / 1e6).toFixed(0)}/${(m.heapTotal / 1e6).toFixed(0)} MB, external ${(m.external / 1e6).toFixed(0)} MB, array buffers ${(m.arrayBuffers / 1e6).toFixed(0)} MB`
+      }
+    }
+    // Done, so nothing takes it for a range to work on (a leased one was set back to planned once).
+    if (this.store) await db.from('ranges').where('id', rangeId).update({ status: 'done', done_at: new Date() })
+    const cpu = process.cpuUsage(cpuStart)
+    const cpuS = (cpu.user + cpu.system) / 1e6
+    const total = t.inflate + t.parse + t.index
+    this.logger.info(`${group} ${start}–${end}: ${files.length} chunks, ${(compressed / 1e6).toFixed(0)} MB compressed, ${(text / 1e6).toFixed(0)} MB text`)
+    this.logger.info(`${out.headers.toLocaleString('en')} headers in ${(total / 1000).toFixed(1)} s (${Math.round(out.headers / (total / 1000)).toLocaleString('en')}/s on one core; CPU ${cpuS.toFixed(1)} s, wall ${((Date.now() - wall) / 1000).toFixed(1)} s)`)
+    this.logger.info(`  inflate ${(t.inflate / 1000).toFixed(1)} s, parse ${(t.parse / 1000).toFixed(1)} s, index ${(t.index / 1000).toFixed(1)} s; peak RSS ${(peakRss / 1e6).toFixed(0)} MB (${peakDetail})`)
+    this.logger.info(`  ${batches} batches of ${this.batchChunks} chunks: ${out.releases} release chunks for ${keys.size} releases (${(out.releases / Math.max(1, keys.size)).toFixed(2)} chunks per release)`)
+    this.logger.info(`  kept: ${out.releases} release chunks, ${out.files} files, ${out.segments} segments, payload ${(out.payload / 1e6).toFixed(1)} MB (${(out.payload / Math.max(1, out.segments)).toFixed(1)} B/segment); ${out.candidates} candidates, ${out.fragments} fragments, ${out.samples} samples`)
+    for (const [k, v] of [...counts].sort((a, b) => b[1] - a[1]).slice(0, 8)) this.logger.info(`  dropped ${k}: ${v.toLocaleString('en')} articles`)
+    if (this.store) this.logger.info(`  stored ${batches} batches in ${(storeMs / 1000).toFixed(1)} s (${(storeMs / Math.max(1, out.releases)).toFixed(2)} ms per release chunk)`)
+  }
+}

commands/edge_node.ts

@@ -0,0 +1,271 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { spawn, type ChildProcess } from 'node:child_process'
+import { availableParallelism } from 'node:os'
+import { promisify } from 'node:util'
+import { gzip, unzip } from 'node:zlib'
+import { RangeIndexer, decodeFrames, isKeepalive, throttle, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
+
+const IDLE_MS = 30_000
+/** A batch is posted once this many files (completed, or cut at an edge) wait, whatever its articles. */
+const MAX_PENDING_FILES = 100_000
+const unzipAsync = promisify(unzip)
+const gzipAsync = promisify(gzip)
+const RETRIES = 8
+
+class LeaseLost extends Error {}
+/** A spooled chunk that doesn't inflate (reported; the coordinator fetches it again). */
+class CorruptChunk extends Error {}
+
+/**
+ * An edge worker (REBUILD.md, part 2): leases a range from the coordinator,
+ * streams its chunks, indexes them, posts a batch every --batch-articles
+ * articles (each moves the range's resume point), and finishes the range.
+ * Needs COORDINATOR_URL and NODE_TOKEN only. On SIGTERM it gives its range
+ * back after the current chunk; the next worker resumes from the last batch.
+ */
+export default class EdgeNode extends BaseCommand {
+  static commandName = 'edge:node'
+  static description = 'Indexes ranges for the coordinator'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.number({ description: 'Articles per posted batch (500k: ~4% of releases get a second chunk; LOG.md 2026-10-09)', default: 500_000 })
+  declare batchArticles: number
+
+  @flags.number({ description: 'Stop after this many ranges (default: keep going)' })
+  declare ranges?: number
+
+  @flags.number({ description: 'Worker processes (one core each; default: EDGE_PROCESSES or 1, 0 = one per core)' })
+  declare processes?: number
+
+  @flags.number({ description: 'Download cap for the whole node in Mbit/s, shared by its processes (default: EDGE_MAX_MBIT or none)' })
+  declare maxMbit?: number
+
+  private stopping = false
+
+  async run() {
+    const url = process.env.COORDINATOR_URL
+    const token = process.env.NODE_TOKEN
+    if (!url || !token) throw new Error('COORDINATOR_URL and NODE_TOKEN are needed')
+    const requested = this.processes ?? Number(process.env.EDGE_PROCESSES ?? 1)
+    const processes = requested === 0 ? availableParallelism() : requested
+    if (processes > 1 && !process.env.EDGE_CHILD) return this.supervise(processes)
+    let stopped: () => void
+    const done = new Promise<void>((resolve) => (stopped = resolve))
+    this.app.terminating(async () => {
+      this.stopping = true
+      await done
+    })
+    const api = new Api(url.replace(/\/$/, ''), token)
+    let ranges = 0
+    try {
+      while (!this.stopping && (!this.ranges || ranges < this.ranges)) {
+        let lease: Lease | null = null
+        try {
+          lease = await api.lease()
+        } catch (error: any) {
+          this.logger.warning(`Lease failed: ${error.message}`)
+        }
+        if (!lease) {
+          if (this.ranges) break
+          await sleep(IDLE_MS)
+          continue
+        }
+        await this.runRange(api, lease)
+        ranges++
+      }
+    } finally {
+      stopped!()
+    }
+    await this.terminate()
+  }
+
+  /**
+   * One worker per process (a worker is single-threaded): starts `count`
+   * children, restarts one that exits unexpectedly, and passes SIGTERM on
+   * (each gives its range back).
+   */
+  private async supervise(count: number) {
+    const cap = this.cap()
+    this.logger.info(`Starting ${count} worker processes${cap ? ` (download cap ${cap} Mbit/s, ${(cap / count).toFixed(1)} each)` : ''}.`)
+    const children = new Set<ChildProcess>()
+    let stopping = false
+    let stopped: () => void
+    const done = new Promise<void>((resolve) => (stopped = resolve))
+    const start = (slot: number) => {
+      const args = [
+        process.argv[1],
+        'edge:node',
+        `--batch-articles=${this.batchArticles}`,
+        ...(this.ranges ? [`--ranges=${this.ranges}`] : []),
+        // Each child gets its share of the node's cap.
+        ...(cap ? [`--max-mbit=${cap / count}`] : []),
+      ]
+      const child = spawn(process.execPath, [...process.execArgv, ...args], { env: { ...process.env, EDGE_CHILD: String(slot) }, stdio: 'inherit' })
+      children.add(child)
+      child.on('exit', (code) => {
+        children.delete(child)
+        if (stopping || (this.ranges && code === 0)) {
+          if (children.size === 0) stopped()
+          return
+        }
+        this.logger.warning(`Worker ${slot} exited (${code}); starting it again in 10 s.`)
+        setTimeout(() => !stopping && start(slot), 10_000)
+      })
+    }
+    for (let slot = 1; slot <= count; slot++) start(slot)
+    this.app.terminating(async () => {
+      stopping = true
+      for (const child of children) child.kill('SIGTERM')
+      await done
+    })
+    await done
+    await this.terminate()
+  }
+
+  /** The node's download cap in Mbit/s, or null. */
+  private cap(): number | null {
+    const mbit = this.maxMbit ?? (process.env.EDGE_MAX_MBIT ? Number(process.env.EDGE_MAX_MBIT) : null)
+    return mbit && mbit > 0 ? mbit : null
+  }
+
+  private async runRange(api: Api, lease: Lease) {
+    const { range, rules } = lease
+    const started = Date.now()
+    const indexer = new RangeIndexer({ group: range.group, start: range.start, end: range.end, resumeAt: range.next, rules })
+    const totals = { headers: 0, releases: 0, candidates: 0, bytes: 0, postMs: 0 }
+    let batchFrom = range.next
+    let reachedEnd = false
+    // Posts go in order, but the worker indexes on meanwhile (a post took 1–16 s; LOG.md 2026-10-09).
+    let posting: Promise<void> = Promise.resolve()
+    const post = (batch: EdgeBatch) => {
+      const previous = posting
+      posting = (async () => {
+        await previous
+        const at = Date.now()
+        const stored = await api.batch(range.id, batch)
+        totals.postMs += Date.now() - at
+        totals.bytes += stored.bytes
+        totals.headers += batch.headers
+        totals.releases += stored.releases
+        totals.candidates += stored.candidates
+      })()
+      // A failed post stops the range at the next chunk (a lost lease: nothing more will be stored).
+      posting.catch((error) => (postError ??= error))
+    }
+    let postError: Error | null = null
+    const handle = async (chunk: { from: number; to: number; text: Promise<Buffer> }) => {
+      if (postError) throw postError
+      let text: Buffer
+      try {
+        text = await chunk.text
+      } catch (error: any) {
+        await posting.catch(() => {})
+        await api.corrupt(range.id, chunk.from)
+        throw new CorruptChunk(`chunk ${chunk.from}–${chunk.to} doesn't inflate (${error.message}); reported, the range goes back`)
+      }
+      indexer.addText(text.toString('utf8'), chunk.to + 1)
+      reachedEnd = chunk.to >= range.end
+      if (reachedEnd) indexer.finish()
+      // Also early when many files wait: a picture group's 500k articles made batches over the server's 256 MB limit (HTTP 413).
+      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles || indexer.pending >= MAX_PENDING_FILES) {
+        post(indexer.take())
+        batchFrom = chunk.to + 1
+      }
+    }
+    try {
+      // Inflating runs on a thread-pool thread: the next chunk inflates while this one is indexed.
+      let pending: { from: number; to: number; text: Promise<Buffer> } | null = null
+      const cap = this.cap()
+      const stream = await api.stream(range.id)
+      for await (const frame of decodeFrames(cap ? throttle(stream, (cap * 1e6) / 8) : stream)) {
+        if (this.stopping) break
+        if (isKeepalive(frame)) continue
+        const next = { from: frame.from, to: frame.to, text: unzipAsync(frame.data) }
+        next.text.catch(() => {})
+        if (pending) await handle(pending)
+        pending = next
+      }
+      if (pending && !this.stopping) await handle(pending)
+      await posting
+      if (!reachedEnd) {
+        // Stopping: back to the queue; the next worker resumes at the last stored batch.
+        this.logger.info(`Range ${range.id}: giving it back.`)
+        await api.call(range.id, 'release')
+        return
+      }
+      await api.call(range.id, 'finish')
+      const seconds = (Date.now() - started) / 1000
+      this.logger.info(
+        `${range.group} ${range.start}–${range.end}: ${totals.headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s ` +
+          `(${Math.round(totals.headers / seconds).toLocaleString('en')}/s), ${totals.releases} release chunks, ${totals.candidates} probe candidates; ` +
+          `posted ${(totals.bytes / 1e6).toFixed(1)} MB in ${(totals.postMs / 1000).toFixed(1)} s.`
+      )
+    } catch (error: any) {
+      await posting.catch(() => {})
+      if (error instanceof LeaseLost || error instanceof CorruptChunk) {
+        this.logger.warning(`Range ${range.id}: ${error.message}`)
+        return
+      }
+      this.logger.error(`Range ${range.id} failed: ${error.message}; giving it back.`)
+      await api.call(range.id, 'release').catch(() => {})
+    }
+  }
+}
+
+/** The coordinator's node API, with retries for network trouble and 5xx. */
+class Api {
+  constructor(
+    private base: string,
+    private token: string
+  ) {}
+
+  private async request(path: string, init: RequestInit = {}): Promise<Response> {
+    for (let attempt = 1; ; attempt++) {
+      try {
+        const response = await fetch(`${this.base}/api/v1/node/${path}`, {
+          ...init,
+          headers: { 'Authorization': `Bearer ${this.token}`, 'Content-Type': 'application/json', 'Accept': 'application/json', ...init.headers },
+        })
+        if (response.status === 409) throw new LeaseLost(((await response.json().catch(() => ({}))) as { error?: string }).error ?? 'lease lost')
+        if (response.status < 500) return response
+        if (attempt >= RETRIES) throw new Error(`${path}: HTTP ${response.status}`)
+      } catch (error) {
+        if (error instanceof LeaseLost || attempt >= RETRIES) throw error
+      }
+      await sleep(Math.min(60_000, 1000 * 2 ** attempt))
+    }
+  }
+
+  async lease(): Promise<Lease | null> {
+    const response = await this.request('lease', { method: 'POST', body: '{}' })
+    if (response.status === 204) return null
+    if (!response.ok) throw new Error(`lease: HTTP ${response.status}`)
+    return response.json() as Promise<Lease>
+  }
+
+  async stream(rangeId: number): Promise<AsyncIterable<Uint8Array>> {
+    const response = await this.request(`ranges/${rangeId}/stream`, { headers: { Accept: 'application/octet-stream' } })
+    if (!response.ok || !response.body) throw new Error(`stream: HTTP ${response.status}`)
+    return response.body as unknown as AsyncIterable<Uint8Array>
+  }
+
+  async batch(rangeId: number, batch: EdgeBatch): Promise<{ releases: number; candidates: number; bytes: number }> {
+    // Gzipped (on a thread-pool thread): chunk payloads are compressed already, the rest compresses well.
+    const body = await gzipAsync(JSON.stringify(toWire(batch)), { level: 1 })
+    const response = await this.request(`ranges/${rangeId}/batches`, { method: 'POST', body, headers: { 'Content-Encoding': 'gzip' } })
+    if (!response.ok) throw new Error(`batch: HTTP ${response.status}`)
+    return { ...((await response.json()) as { releases: number; candidates: number }), bytes: body.length }
+  }
+
+  async corrupt(rangeId: number, from: number) {
+    const response = await this.request(`ranges/${rangeId}/corrupt`, { method: 'POST', body: JSON.stringify({ from }) })
+    if (!response.ok) throw new Error(`corrupt: HTTP ${response.status}`)
+  }
+
+  async call(rangeId: number, action: 'finish' | 'release' | 'heartbeat') {
+    const response = await this.request(`ranges/${rangeId}/${action}`, { method: 'POST', body: '{}' })
+    if (!response.ok) throw new Error(`${action}: HTTP ${response.status}`)
+  }
+}

commands/groups_add.ts

@@ -0,0 +1,30 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Planner from '#services/planner'
+import NntpPool from '#services/nntp_pool'
+
+/** Adds groups to index, after the existing ones in priority (owner rule). */
+export default class GroupsAdd extends BaseCommand {
+  static commandName = 'groups:add'
+  static description = 'Adds groups to index (by name, or the largest by LIST ACTIVE)'
+  static options: CommandOptions = { startApp: true }
+
+  @args.spread({ description: 'Group names', required: false })
+  declare names?: string[]
+
+  @flags.number({ description: 'The largest N groups matching --pattern' })
+  declare top?: number
+
+  @flags.string({ description: 'LIST ACTIVE pattern for --top', default: 'alt.binaries.*' })
+  declare pattern: string
+
+  async run() {
+    try {
+      const names = this.top ? await Planner.largestGroups(this.pattern, this.top) : (this.names ?? [])
+      const added = await Planner.addGroups(names)
+      this.logger.info(`${added} of ${names.length} groups added.`)
+    } finally {
+      await NntpPool.close()
+    }
+  }
+}

commands/nodes_create.ts

@@ -0,0 +1,19 @@
+import { BaseCommand, args } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Node from '#models/node'
+
+/** Registers an edge worker and prints its token (shown once). */
+export default class NodesCreate extends BaseCommand {
+  static commandName = 'nodes:create'
+  static description = 'Registers an edge worker and prints its token'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'Node name' })
+  declare name: string
+
+  async run() {
+    const node = await Node.firstOrCreate({ name: this.name }, { name: this.name })
+    const token = await Node.accessTokens.create(node)
+    this.logger.info(`Node ${node.name} (#${node.id}). Token: ${token.value!.release()}`)
+  }
+}

commands/probe_run.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Prober } from '#services/prober'
+
+/** Phase 2's prober (REBUILD.md, part 6): reads the probe queue's articles and stores what they show. */
+export default class ProbeRun extends BaseCommand {
+  static commandName = 'probe:run'
+  static description = "Reads the probe queue's articles (PROBE_CONNECTIONS) and stores what they show"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  async run() {
+    const prober = new Prober()
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      this.logger.info('Stopping: finishing the articles in flight.')
+      prober.stop()
+      await finished
+    })
+    await prober.run()
+    this.logger.info('Prober stopped.')
+    done!()
+    await this.terminate()
+  }
+}

commands/ranges_plan.ts

@@ -0,0 +1,23 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Planner from '#services/planner'
+import NntpPool from '#services/nntp_pool'
+
+/** Cuts new groups' history into ranges, or (--tail) plans one round of the live tail. */
+export default class RangesPlan extends BaseCommand {
+  static commandName = 'ranges:plan'
+  static description = "Plans ranges for new groups' history, or one round of the live tail"
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Plan the live tail once (what coordinator:run does every few minutes)' })
+  declare tail: boolean
+
+  async run() {
+    try {
+      const { groups, ranges } = this.tail ? await Planner.planTail() : await Planner.planAll()
+      this.logger.info(`${ranges} ${this.tail ? 'tail ' : ''}ranges planned over ${groups} groups.`)
+    } finally {
+      await NntpPool.close()
+    }
+  }
+}

commands/releases_duplicates.ts

@@ -0,0 +1,82 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+
+/**
+ * How often a release holds a file name more than once (decoded from its
+ * chunks): copies of one upload posted again (multi-posting, reposts that
+ * share a key) or different files that share a name. Copies of the same
+ * size are counted as the same file posted again.
+ */
+export default class ReleasesDuplicates extends BaseCommand {
+  static commandName = 'releases:duplicates'
+  static description = 'Counts file names a release holds more than once, in a sample'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
+  declare sample: number
+
+  async run() {
+    const ids: string[] = (
+      await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])
+    ).rows.map((r: { id: string }) => r.id)
+    const kind = (name: string) =>
+      /\.(jpe?g|png|gif|bmp|webp)$/i.test(name) ? 'image'
+      : /\.(mp3|flac|m4a|ogg|wav)\b/i.test(name) ? 'audio'
+      : /\.(par2)$/i.test(name) ? 'par2'
+      : /\.(rar|r\d\d|7z|zip|\d{3})$/i.test(name) ? 'archive'
+      : /\.(pdf|epub|mobi|cbr|cbz|djvu|chm)$/i.test(name) ? 'document'
+      : /\.(nfo|sfv|nzb|txt|diz)$/i.test(name) ? 'sidecar'
+      : 'other'
+    let releases = 0
+    let withDuplicates = 0
+    let files = 0
+    let extra = 0
+    let completeOnlyByCopies = 0
+    const byKind = new Map<string, number>()
+    const how = { sameFile: 0, differentFile: 0, sameDay: 0, laterRepost: 0 }
+    const examples: string[] = []
+    for (const id of ids) {
+      const release = await db.from('releases').where('id', id).select('name', 'file_total', 'files').first()
+      const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+      const byName = new Map<string, Map<string, { parts: number; bytes: number; at: number | null }>>()
+      for (const c of chunks) {
+        for (const f of decodeChunk(c.payload)) {
+          const copies = byName.get(f.name) ?? new Map()
+          copies.set(f.segments[0]?.[1] ?? '', { parts: f.segments.length, bytes: f.segments.reduce((n, [, , b]) => n + b, 0), at: f.postedAt })
+          byName.set(f.name, copies)
+        }
+      }
+      releases++
+      let dup = false
+      for (const [name, copies] of byName) {
+        files += copies.size
+        if (copies.size < 2) continue
+        dup = true
+        extra += copies.size - 1
+        byKind.set(kind(name), (byKind.get(kind(name)) ?? 0) + copies.size - 1)
+        const list = [...copies.values()]
+        // The same file again: as many parts, within 3% in bytes (article sizes include headers).
+        const same = list.every((c) => c.parts === list[0].parts && Math.abs(c.bytes - list[0].bytes) <= 0.03 * list[0].bytes)
+        if (same) how.sameFile++
+        else how.differentFile++
+        const times = list.map((c) => c.at).filter((t): t is number => t !== null)
+        if (times.length > 1 && Math.max(...times) - Math.min(...times) > 86_400_000) how.laterRepost++
+        else how.sameDay++
+        if (!same && examples.length < 8) examples.push(`${id} ${release.name.slice(0, 60)} | ${name.slice(0, 60)} | ${list.map((c) => `${c.parts}p ${(c.bytes / 1e6).toFixed(1)}MB`).join(', ')}`)
+      }
+      if (!dup) continue
+      withDuplicates++
+      const total = Number(release.file_total ?? 0)
+      if (total && Number(release.files) >= total && byName.size < total) completeOnlyByCopies++
+    }
+    const pct = (n: number, of: number) => `${((100 * n) / Math.max(1, of)).toFixed(2)}%`
+    this.logger.info(`${releases} releases with 2+ files sampled`)
+    this.logger.info(`  with a file name more than once: ${withDuplicates} (${pct(withDuplicates, releases)})`)
+    this.logger.info(`  extra copies: ${extra} of ${files} files (${pct(extra, files)}); by type: ${[...byKind].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${k} ${n}`).join(', ')}`)
+    this.logger.info(`  duplicated names: same file again ${how.sameFile}, different (parts or size) ${how.differentFile}; copies the same day ${how.sameDay}, a day or more apart ${how.laterRepost}`)
+    this.logger.info(`  'complete' only by counting copies: ${completeOnlyByCopies}`)
+    for (const e of examples) this.logger.info(`  different: ${e}`)
+  }
+}

commands/releases_judge.ts

@@ -0,0 +1,38 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { ReleaseJudge, RULES } from '#services/release_judge'
+
+/** The junk judge (REBUILD.md, junk): releases judged after their probes; verdicts hide them from search. */
+export default class ReleasesJudge extends BaseCommand {
+  static commandName = 'releases:judge'
+  static description = 'Judges releases by what their probes showed (dead, fake, repeated, locked, unusable), hourly'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  @flags.boolean({ description: 'With --once: count only, write no verdicts' })
+  declare dry: boolean
+
+  async run() {
+    const judge = new ReleaseJudge()
+    if (this.once) {
+      const round = await judge.round(this.dry)
+      for (const rule of RULES) {
+        const r = round.rules[rule.name]
+        if (r) this.logger.info(`${rule.name}: ${r.matched} (+${r.added} −${r.removed}) — ${rule.says}`)
+      }
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      judge.stop()
+      await finished
+    })
+    await judge.run()
+    this.logger.info('Judge stopped.')
+    done!()
+  }
+}

commands/releases_merge.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Par2Merger } from '#services/par2_merger'
+
+/** The par2 merge (REBUILD.md, phase 2): releases whose files one par2 set describes become one. */
+export default class ReleasesMerge extends BaseCommand {
+  static commandName = 'releases:merge'
+  static description = 'Merges the releases whose files one par2 set describes (what the prober read), every minute'
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round over every set, then exit' })
+  declare once: boolean
+
+  async run() {
+    const merger = new Par2Merger()
+    if (this.once) {
+      const round = await merger.round()
+      this.logger.info(`${round.sets} sets checked: ${round.releases} releases merged into ${round.merges}.`)
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      merger.stop()
+      await finished
+    })
+    await merger.run()
+    this.logger.info('Merger stopped.')
+    done!()
+    await this.terminate()
+  }
+}

commands/releases_regroup.ts

@@ -0,0 +1,86 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+import { releaseIdentity } from '#lib/release/index'
+
+/**
+ * What lib/release's identity makes of releases already in the catalog: a
+ * sample of releases (by source), their files decoded from the chunks and
+ * each given its release key again. Shows how many would now split, and
+ * into what: for checking an identity change on real data.
+ */
+export default class ReleasesRegroup extends BaseCommand {
+  static commandName = 'releases:regroup'
+  static description = 'Shows how the current release identity would regroup a sample of the catalog'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled per source', default: 2000 })
+  declare sample: number
+
+  @flags.array({ description: 'Release ids to show in full' })
+  declare show?: string[]
+
+  @flags.array({ description: 'Groups whose indexed releases are re-keyed together (merges show too)' })
+  declare group?: string[]
+
+  async run() {
+    for (const name of this.group ?? []) {
+      const { rows } = await db.rawQuery(
+        `SELECT DISTINCT c.release_id FROM release_chunks c JOIN ranges r ON r.id = c.range_id JOIN groups g ON g.id = r.group_id WHERE g.name = ?`,
+        [name]
+      )
+      const ids = rows.map((r: { release_id: string }) => r.release_id)
+      const all = new Map<string, { name: string; source: string; files: number; subject?: string }>()
+      for (const id of ids) {
+        for (const [key, g] of await this.regroup(Number(id))) {
+          const known = all.get(key)
+          if (known) known.files += g.files
+          else all.set(key, { ...g })
+        }
+      }
+      const sizes = [...all.values()].map((g) => g.files)
+      const single = sizes.filter((n) => n === 1).length
+      this.logger.info(`${name}: ${ids.length} releases now, ${all.size} with the current rule (${single} of one file)`)
+      for (const g of [...all.values()].sort((a, b) => b.files - a.files).slice(0, 6)) this.logger.info(`  ${g.files} files: ${g.name.slice(0, 120)} (${g.source})`)
+      const singles = [...all.values()].filter((g) => g.files === 1)
+      for (let i = 0; i < Math.min(8, singles.length); i++) this.logger.info(`  single: ${singles[Math.floor((i * singles.length) / 8)].subject?.slice(0, 150)}`)
+    }
+    if (this.group?.length) return
+    for (const id of this.show ?? []) {
+      const groups = await this.regroup(Number(id))
+      this.logger.info(`${id}: ${groups.size} release(s) now`)
+      for (const [, g] of groups) this.logger.info(`  ${g.files} files: ${g.name} (${g.source})`)
+    }
+    if (this.show?.length) return
+    for (const source of ['subject', 'post', 'stem']) {
+      const rows = await db.from('releases').where('source', source).where('files', '>', 1).orderByRaw('random()').limit(this.sample).select('id', 'name', 'files')
+      let split = 0
+      let pieces = 0
+      const examples: string[] = []
+      for (const r of rows) {
+        const groups = await this.regroup(Number(r.id))
+        if (groups.size <= 1) continue
+        split++
+        pieces += groups.size
+        if (examples.length < 5) examples.push(`${r.id} ${r.name} (${r.files} files) → ${groups.size}: ${[...groups.values()].slice(0, 3).map((g) => `${g.name} [${g.files}]`).join(' | ')}`)
+      }
+      this.logger.info(`${source}: ${rows.length} sampled, ${split} would split (into ${pieces} releases)`)
+      for (const e of examples) this.logger.info(`  ${e.slice(0, 260)}`)
+    }
+  }
+
+  private async regroup(id: number) {
+    const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+    const groups = new Map<string, { name: string; source: string; files: number; subject?: string }>()
+    for (const c of chunks) {
+      for (const f of decodeChunk(c.payload)) {
+        const r = releaseIdentity({ poster: f.poster, subject: f.subject, name: f.name, weak: false })!
+        const g = groups.get(r.key)
+        if (g) g.files++
+        else groups.set(r.key, { name: r.name, source: r.source, files: 1, subject: f.subject })
+      }
+    }
+    return groups
+  }
+}

commands/releases_upload_gaps.ts

@@ -0,0 +1,58 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { decodeChunk } from '#lib/chunks/index'
+
+/**
+ * For telling uploads apart (re-uploads under one release key): how long
+ * one upload takes to post (the span of a release's files' dates, in
+ * releases without a file name twice), and how far apart the copies of a
+ * file name are in releases that have one twice.
+ */
+export default class ReleasesUploadGaps extends BaseCommand {
+  static commandName = 'releases:upload-gaps'
+  static description = 'Posting spans of uploads and gaps between re-uploads, in a sample'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.number({ description: 'Releases sampled (with more than one file)', default: 20000 })
+  declare sample: number
+
+  async run() {
+    const ids: string[] = (await db.rawQuery(`SELECT id FROM releases TABLESAMPLE SYSTEM (5) WHERE files > 1 LIMIT ?`, [this.sample])).rows.map((r: { id: string }) => r.id)
+    const bins = [10, 60, 180, 360, 720, 1440, 4320, Number.POSITIVE_INFINITY]
+    const label = (m: number) => (m === Number.POSITIVE_INFINITY ? '>3d' : m >= 1440 ? `≤${m / 1440}d` : m >= 60 ? `≤${m / 60}h` : `≤${m}min`)
+    const spans = new Array(bins.length).fill(0)
+    const gaps = new Array(bins.length).fill(0)
+    const bin = (minutes: number) => bins.findIndex((b) => minutes <= b)
+    for (const id of ids) {
+      const chunks = await db.from('release_chunks').where('release_id', id).select('payload')
+      const byName = new Map<string, number[]>()
+      const times: number[] = []
+      for (const c of chunks) {
+        for (const f of decodeChunk(c.payload)) {
+          if (f.postedAt === null) continue
+          times.push(f.postedAt)
+          byName.set(f.name, [...(byName.get(f.name) ?? []), f.postedAt])
+        }
+      }
+      if (times.length < 2) continue
+      const twice = [...byName.values()].filter((t) => t.length > 1)
+      if (twice.length === 0) {
+        spans[bin((Math.max(...times) - Math.min(...times)) / 60_000)]++
+        continue
+      }
+      // The gap between a name's copies (sorted; each step), so a release reposted 3 times counts 2 gaps per name.
+      for (const t of twice) {
+        t.sort((a, b) => a - b)
+        for (let i = 1; i < t.length; i++) gaps[bin((t[i] - t[i - 1]) / 60_000)]++
+      }
+    }
+    const show = (counts: number[]) => {
+      const total = counts.reduce((a, b) => a + b, 0)
+      let cumulative = 0
+      return bins.map((b, i) => `${label(b)} ${((100 * (cumulative += counts[i])) / Math.max(1, total)).toFixed(1)}%`).join(', ') + ` (n=${total})`
+    }
+    this.logger.info(`one upload's posting span (cumulative): ${show(spans)}`)
+    this.logger.info(`gap between copies of a name (cumulative): ${show(gaps)}`)
+  }
+}

commands/rules_signatures.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { readFile } from 'node:fs/promises'
+import RuleSets from '#services/rule_sets'
+import { signatureKey } from '#lib/rules/index'
+
+/**
+ * Sets the dropped upload signatures (REBUILD.md, rules and recovery) from a
+ * file of "tool<TAB>group" lines, e.g. voyager 1's learned ones:
+ *
+ *   psql … -At -F $'\t' -c "SELECT tool, group_name FROM upload_signatures WHERE dropped"
+ *
+ * Publishes a new rule set version (ranges record the version they ran with).
+ */
+export default class RulesSignatures extends BaseCommand {
+  static commandName = 'rules:signatures'
+  static description = 'Publishes a rule set with the dropped upload signatures from a file'
+  static options: CommandOptions = { startApp: true }
+
+  @args.string({ description: 'File of "tool<TAB>group" lines (more columns and # lines are ignored), e.g. database/data/voyager1_signatures.tsv' })
+  declare file: string
+
+  @flags.string({ description: 'Note stored with the rule set version' })
+  declare note?: string
+
+  @flags.boolean({ description: 'Add to the current signatures instead of replacing them' })
+  declare add: boolean
+
+  async run() {
+    const lines = (await readFile(this.file, 'utf8')).split('\n').map((l) => l.trim()).filter((l) => l && !l.startsWith('#'))
+    const signatures = new Set<string>()
+    for (const line of lines) {
+      const [tool, group] = line.split('\t')
+      if (!tool || !group) throw new Error(`Not "tool<TAB>group": ${line}`)
+      signatures.add(signatureKey(tool, group))
+    }
+    const { version: _v, priorities: _p, ...current } = await RuleSets.current()
+    const before = new Set(current.droppedSignatures)
+    const next = this.add ? new Set([...before, ...signatures]) : signatures
+    if (next.size === before.size && [...next].every((s) => before.has(s))) {
+      this.logger.info(`Unchanged: ${before.size} dropped signatures already.`)
+      return
+    }
+    const version = await RuleSets.publish(
+      { ...current, droppedSignatures: [...next].sort() },
+      this.note ?? `${next.size} dropped signatures from ${this.file}`
+    )
+    const added = [...next].filter((s) => !before.has(s)).length
+    const removed = [...before].filter((s) => !next.has(s)).length
+    this.logger.info(`Rule set ${version}: ${next.size} dropped signatures (${added} added, ${removed} removed).`)
+  }
+}

commands/spots_inspect.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { SpotInspector } from '#services/spot_inspector'
+
+/** The spot inspector (REBUILD.md, spots): each spot's NZB read, its files listed and matched to releases. */
+export default class SpotsInspect extends BaseCommand {
+  static commandName = 'spots:inspect'
+  static description = "Reads each listed spot's NZB, lists its files and links them to the releases they match"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  async run() {
+    const inspector = new SpotInspector()
+    if (this.once) {
+      const round = await inspector.round()
+      this.logger.info(`${round.inspected} spots inspected, ${round.matched} matched releases, ${round.standalone} of their own.`)
+      inspector.stop()
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      inspector.stop()
+      await finished
+    })
+    await inspector.run()
+    this.logger.info('Spot inspector stopped.')
+    done!()
+  }
+}

commands/spots_read.ts

@@ -0,0 +1,33 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Spotter } from '#services/spotter'
+
+/** The spotter (REBUILD.md, spots): Spotnet's spots from free.pt's headers into `spots`, on one connection. */
+export default class SpotsRead extends BaseCommand {
+  static commandName = 'spots:read'
+  static description = "Reads Spotnet's spots (free.pt) into the spots table, verified, every few minutes"
+  static options: CommandOptions = { startApp: true, staysAlive: true }
+
+  @flags.boolean({ description: 'One round, then exit' })
+  declare once: boolean
+
+  async run() {
+    const spotter = new Spotter()
+    if (this.once) {
+      const round = await spotter.round()
+      this.logger.info(`${round.read} articles read: ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
+      spotter.stop()
+      await this.terminate()
+      return
+    }
+    let done: () => void
+    const finished = new Promise<void>((resolve) => (done = resolve))
+    this.app.terminating(async () => {
+      spotter.stop()
+      await finished
+    })
+    await spotter.run()
+    this.logger.info('Spotter stopped.')
+    done!()
+  }
+}

commands/workers/Body.ts

@@ -1,97 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import { Worker } from 'bullmq'
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import queueConfig from '#config/queue'
-import Shutdown from '#services/Shutdown'
-import QueueService from '#services/QueueService'
-import env from '#start/env'
-import NntpService from '#services/NntpService'
-import IngestService, { segmentFromYenc } from '#services/IngestService'
-import { parseYencMeta, YencError } from '#lib/yenc/index'
-import { firstChunkMd5 } from '#services/ContentService'
-
-export default class BodyWorker extends BaseCommand {
-  public static commandName = 'worker:body'
-  public static description = 'Starts a worker to process article bodies for yEnc metadata.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  public async run() {
-    this.logger.info('Starting body worker...')
-
-    const pool = NntpService
-
-    const worker = new Worker('body-queue', async (job) => {
-      const { header, group, articleNumber } = job.data
-      const messageId = header['message-id']
-      this.logger.debug(`Processing header with unparsable subject: ${header.subject}`)
-
-      let conn
-      let failed = false
-      try {
-        conn = await pool.acquire()
-        const bodyBuffer = await pool.fetchBody(conn, messageId)
-
-        let meta
-        try {
-          meta = parseYencMeta(bodyBuffer)
-        } catch (parseError: any) {
-          if (parseError instanceof YencError && bodyBuffer.indexOf('=ybegin ') === -1) {
-            this.logger.debug(`Not a yEnc post, skipping: ${messageId}`)
-            return
-          }
-          this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`)
-          const debugDir = path.join(this.app.appRoot.pathname, 'debug')
-          await fs.mkdir(debugDir, { recursive: true })
-          const timestamp = new Date().toISOString().replace(/:/g, '-')
-          const dumpFile = path.join(debugDir, `body-error-${timestamp}-${messageId.replace(/[<>]/g, '')}.bin`)
-          await fs.writeFile(dumpFile, bodyBuffer)
-          this.logger.error(`Problematic body buffer saved to: ${dumpFile}`)
-          throw parseError
-        }
-
-        pool.release(conn)
-        conn = undefined
-
-        const segment = segmentFromYenc(header, group, meta, articleNumber)
-        if (segment?.number === 1) segment.md5_16k = firstChunkMd5(bodyBuffer, meta.begin.size)
-        if (!segment) {
-          this.logger.warning(`Inconsistent yEnc part numbering in ${messageId}, skipping.`)
-          return
-        }
-        const completed = await IngestService.ingest([segment])
-        this.logger.debug(
-          `Stored part ${segment.number}/${segment.totalParts} of "${segment.name}"` +
-            (completed.length > 0 ? ' (file complete)' : '')
-        )
-      } catch (error: any) {
-        failed = true
-        this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`)
-        throw error
-      } finally {
-        if (conn) {
-          pool.release(conn, failed)
-        }
-      }
-    }, {
-      connection: queueConfig.connection,
-      // Each job holds one NNTP connection while downloading, so keep this <= the pool size.
-      concurrency: env.get('BODY_CONCURRENCY', 1),
-    })
-
-    worker.on('failed', (job, err) => {
-      this.logger.error(`Body job ${job?.id} failed: ${err.message}`)
-    })
-
-    this.logger.info('Body worker started and listening for jobs.')
-    await Shutdown.wait()
-    this.logger.info('Shutting down: finishing active jobs...')
-    await worker.close()
-    await NntpService.shutdown()
-    await QueueService.closeAll()
-  }
-}

commands/workers/Collection.ts

@@ -1,76 +0,0 @@
-import { BaseCommand } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import { Worker } from 'bullmq'
-import queueConfig from '#config/queue'
-import env from '#start/env'
-import Shutdown from '#services/Shutdown'
-import QueueService from '#services/QueueService'
-import NntpService from '#services/NntpService'
-import ContentService from '#services/ContentService'
-import File from '#models/file'
-
-export default class CollectionWorker extends BaseCommand {
-  public static commandName = 'worker:collection'
-  public static description = 'Starts a worker to process file collections (e.g., RAR archives).'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  public async run() {
-    this.logger.info('Starting collection worker...')
-
-    const worker = new Worker('collection-queue', async (job) => {
-      const { fileId } = job.data
-      this.logger.debug(`Processing file ID ${fileId} for collection.`)
-
-      const file = await File.find(fileId)
-      if (!file) {
-        this.logger.error(`File with ID ${fileId} not found in the database.`)
-        return
-      }
-
-      try {
-        const content = await ContentService.inspect(fileId)
-        if (content?.par2) {
-          const { par2 } = content
-          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
-        } else if (content?.archive) {
-          const { info, releaseRenamed, grouped } = content.archive
-          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
-            .filter(Boolean)
-            .join(', ')
-          this.logger.info(
-            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
-              (grouped ? ', grouped into a release' : '') +
-              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
-          )
-        }
-        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
-        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
-        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
-        if (content?.nzb) {
-          const n = content.nzb
-          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
-        }
-      } catch (error: any) {
-        this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
-      }
-    }, {
-      connection: queueConfig.connection,
-      // Each job holds one NNTP connection while downloading; keep <= the pool size.
-      concurrency: env.get('COLLECTION_CONCURRENCY', 1),
-    })
-
-    worker.on('failed', (job, err) => {
-      this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
-    })
-
-    this.logger.info('Collection worker started and listening for jobs.')
-    await Shutdown.wait()
-    this.logger.info('Shutting down: finishing active jobs...')
-    await worker.close()
-    await NntpService.shutdown()
-    await QueueService.closeAll()
-  }
-}

commands/workers/Deferred.ts

@@ -1,57 +0,0 @@
-import { BaseCommand, flags } from '@adonisjs/core/ace'
-import { CommandOptions } from '@adonisjs/core/types/ace'
-import env from '#start/env'
-import Shutdown from '#services/Shutdown'
-import NntpService from '#services/NntpService'
-import QueueService from '#services/QueueService'
-import DeferredService from '#services/DeferredService'
-
-const IDLE_MS = 60_000
-
-/**
- * Works through `deferred_files` (CONCEPT.md D28): one body per file, and
- * small files completely. Not started by default (compose profile
- * `deferred`); run it when there is bandwidth to spare.
- */
-export default class DeferredWorker extends BaseCommand {
-  public static commandName = 'worker:deferred'
-  public static description = 'Samples deferred files (e.g. ngPost uploads): one article body per file.'
-
-  public static options: CommandOptions = {
-    startApp: true,
-  }
-
-  @flags.number({ description: 'Stop after this many files (default: keep running)' })
-  declare limit?: number
-
-  public async run() {
-    const concurrency = env.get('DEFERRED_CONCURRENCY', 4)
-    this.logger.info(`Deferred worker started (concurrency ${concurrency}).`)
-    const counts: Record<string, number> = {}
-    let done = 0
-    try {
-      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
-        const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
-        if (batch.length === 0) {
-          if (this.limit) break
-          await Shutdown.sleep(IDLE_MS)
-          continue
-        }
-        let next = 0
-        await Promise.all(
-          Array.from({ length: concurrency }, async () => {
-            while (next < batch.length) {
-              const status = await DeferredService.process(batch[next++])
-              counts[status] = (counts[status] ?? 0) + 1
-            }
-          })
-        )
-        done += batch.length
-        this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
-      }
-    } finally {
-      await NntpService.shutdown()
-      await QueueService.closeAll()
-    }
-  }
-}