Add Gitea Actions: test every push, deploy master to homelab

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.adonisjs/client/data.d.ts

@@ -0,0 +1,15 @@
+/**
+ * This file is automatically generated.
+ * DO NOT EDIT manually
+ */
+
+/// <reference path="./manifest.d.ts" />
+import type { InferData, InferVariants } from '@adonisjs/core/types/transformers'
+import type UserTransformer from '#transformers/user_transformer'
+
+export namespace Data {
+  export type User = InferData<UserTransformer>
+  export namespace User {
+    export type Variants = InferVariants<UserTransformer>
+  }
+}

.adonisjs/client/manifest.d.ts

@@ -0,0 +1,14 @@
+/**
+ * This file is automatically generated.
+ * DO NOT EDIT manually
+ */
+
+/// <reference path="../../adonisrc.ts" />
+/// <reference path="../../config/auth.ts" />
+/// <reference path="../../config/gc.ts" />
+/// <reference path="../../config/hash.ts" />
+/// <reference path="../../config/indexer.ts" />
+/// <reference path="../../config/logger.ts" />
+/// <reference path="../../config/nntp.ts" />
+/// <reference path="../../config/queue.ts" />
+/// <reference path="../../config/redis.ts" />

.adonisjs/client/registry/index.ts

@@ -0,0 +1,47 @@
+/* eslint-disable prettier/prettier */
+import type { AdonisEndpoint } from '@tuyau/core/types'
+import type { Registry } from './schema.d.ts'
+import type { ApiDefinition } from './tree.d.ts'
+
+const placeholder: any = {}
+
+const routes = {
+  'auth.new_account.store': {
+    methods: ["POST"],
+    pattern: '/api/v1/auth/signup',
+    tokens: [{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],
+    types: placeholder as Registry['auth.new_account.store']['types'],
+  },
+  'auth.access_tokens.store': {
+    methods: ["POST"],
+    pattern: '/api/v1/auth/login',
+    tokens: [{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],
+    types: placeholder as Registry['auth.access_tokens.store']['types'],
+  },
+  'profile.profile.show': {
+    methods: ["GET","HEAD"],
+    pattern: '/api/v1/account/profile',
+    tokens: [{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],
+    types: placeholder as Registry['profile.profile.show']['types'],
+  },
+  'profile.access_tokens.destroy': {
+    methods: ["POST"],
+    pattern: '/api/v1/account/logout',
+    tokens: [{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],
+    types: placeholder as Registry['profile.access_tokens.destroy']['types'],
+  },
+} as const satisfies Record<string, AdonisEndpoint>
+
+export { routes }
+
+export const registry = {
+  routes,
+  $tree: {} as ApiDefinition,
+}
+
+declare module '@tuyau/core/types' {
+  export interface UserRegistry {
+    routes: typeof routes
+    $tree: ApiDefinition
+  }
+}

.adonisjs/client/registry/schema.d.ts

@@ -0,0 +1,58 @@
+/* eslint-disable prettier/prettier */
+/// <reference path="../manifest.d.ts" />
+
+import type { ExtractBody, ExtractErrorResponse, ExtractQuery, ExtractQueryForGet, ExtractResponse } from '@tuyau/core/types'
+import type { InferInput, SimpleError } from '@vinejs/vine/types'
+
+export type ParamValue = string | number | bigint | boolean
+
+export interface Registry {
+  'auth.new_account.store': {
+    methods: ["POST"]
+    pattern: '/api/v1/auth/signup'
+    types: {
+      body: ExtractBody<InferInput<(typeof import('#validators/user').signupValidator)>>
+      paramsTuple: []
+      params: {}
+      query: ExtractQuery<InferInput<(typeof import('#validators/user').signupValidator)>>
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/new_account_controller').default['store']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/new_account_controller').default['store']>>> | { status: 422; response: { errors: SimpleError[] } }
+    }
+  }
+  'auth.access_tokens.store': {
+    methods: ["POST"]
+    pattern: '/api/v1/auth/login'
+    types: {
+      body: ExtractBody<InferInput<(typeof import('#validators/user').loginValidator)>>
+      paramsTuple: []
+      params: {}
+      query: ExtractQuery<InferInput<(typeof import('#validators/user').loginValidator)>>
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['store']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['store']>>> | { status: 422; response: { errors: SimpleError[] } }
+    }
+  }
+  'profile.profile.show': {
+    methods: ["GET","HEAD"]
+    pattern: '/api/v1/account/profile'
+    types: {
+      body: {}
+      paramsTuple: []
+      params: {}
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/profile_controller').default['show']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/profile_controller').default['show']>>>
+    }
+  }
+  'profile.access_tokens.destroy': {
+    methods: ["POST"]
+    pattern: '/api/v1/account/logout'
+    types: {
+      body: {}
+      paramsTuple: []
+      params: {}
+      query: {}
+      response: ExtractResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['destroy']>>>
+      errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['destroy']>>>
+    }
+  }
+}

.adonisjs/client/registry/tree.d.ts

@@ -0,0 +1,21 @@
+/* eslint-disable prettier/prettier */
+import type { routes } from './index.ts'
+
+export interface ApiDefinition {
+  auth: {
+    newAccount: {
+      store: typeof routes['auth.new_account.store']
+    }
+    accessTokens: {
+      store: typeof routes['auth.access_tokens.store']
+    }
+  }
+  profile: {
+    profile: {
+      show: typeof routes['profile.profile.show']
+    }
+    accessTokens: {
+      destroy: typeof routes['profile.access_tokens.destroy']
+    }
+  }
+}

.adonisjs/server/controllers.ts

@@ -0,0 +1,12 @@
+/**
+ * This file is automatically generated.
+ * DO NOT EDIT manually
+ */
+
+export const controllers = {
+  AccessTokens: () => import('#controllers/access_tokens_controller'),
+  NewAccount: () => import('#controllers/new_account_controller'),
+  Newznab: () => import('#controllers/newznab_controller'),
+  Profile: () => import('#controllers/profile_controller'),
+  Releases: () => import('#controllers/releases_controller'),
+}

.adonisjs/server/events.ts

@@ -0,0 +1,6 @@
+/**
+ * This file is automatically generated.
+ * DO NOT EDIT manually
+ */
+
+export const events = {}

.adonisjs/server/listeners.ts

@@ -0,0 +1,6 @@
+/**
+ * This file is automatically generated.
+ * DO NOT EDIT manually
+ */
+
+export const listeners = {}

.adonisjs/server/routes.d.ts

@@ -0,0 +1,38 @@
+import '@adonisjs/core/types/http'
+
+type ParamValue = string | number | bigint | boolean
+
+export type ScannedRoutes = {
+  ALL: {
+    'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
+    'auth.new_account.store': { paramsTuple?: []; params?: {} }
+    'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
+    'profile.profile.show': { paramsTuple?: []; params?: {} }
+    'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
+  }
+  GET: {
+    'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
+    'profile.profile.show': { paramsTuple?: []; params?: {} }
+  }
+  HEAD: {
+    'releases.index': { paramsTuple?: []; params?: {} }
+    'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
+    'releases.stats': { paramsTuple?: []; params?: {} }
+    'profile.profile.show': { paramsTuple?: []; params?: {} }
+  }
+  POST: {
+    'auth.new_account.store': { paramsTuple?: []; params?: {} }
+    'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
+    'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
+  }
+}
+declare module '@adonisjs/core/types/http' {
+  export interface RoutesList extends ScannedRoutes {}
+}

.adonisjs/server/routes.json

@@ -0,0 +1 @@
+{"root":[{"domain":"root","pattern":"/api/v1/releases","matchers":{},"tokens":[{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],"meta":{"params":[]},"name":"releases.index","handler":{"method":"index","reference":[null,"index"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id","type":1,"val":"id","end":"","matcher":{}}],"meta":{"params":["id"]},"name":"releases.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id/nzb","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":"","matcher":{}},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],"meta":{"params":["id"]},"name":"releases.nzb","handler":{"method":"nzb","reference":[null,"nzb"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/stats","matchers":{},"tokens":[{"old":"/api/v1/stats","type":0,"val":"api","end":""},{"old":"/api/v1/stats","type":0,"val":"v1","end":""},{"old":"/api/v1/stats","type":0,"val":"stats","end":""}],"meta":{"params":[]},"name":"releases.stats","handler":{"method":"stats","reference":[null,"stats"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/signup","matchers":{},"tokens":[{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],"meta":{"params":[]},"name":"auth.new_account.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/new_account_controller')","name":"NewAccount"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/login","matchers":{},"tokens":[{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],"meta":{"params":[]},"name":"auth.access_tokens.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/profile","matchers":{},"tokens":[{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],"meta":{"params":[]},"name":"profile.profile.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/profile_controller')","name":"Profile"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/logout","matchers":{},"tokens":[{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],"meta":{"params":[]},"name":"profile.access_tokens.destroy","handler":{"method":"destroy","reference":[null,"destroy"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}}]}

.dockerignore

@@ -0,0 +1,11 @@
+node_modules
+build
+tmp
+debug
+.git
+.idea
+.vscode
+.env
+.env.*
+!.env.example
+*.log

.editorconfig

@@ -0,0 +1,22 @@
+# http://editorconfig.org
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.json]
+insert_final_newline = unset
+
+[**.min.js]
+indent_style = unset
+insert_final_newline = unset
+
+[MakeFile]
+indent_style = space
+
+[*.md]
+trim_trailing_whitespace = false

.env.example

@@ -0,0 +1,20 @@
+# Node
+TZ=UTC
+PORT=3333
+HOST=localhost
+NODE_ENV=development
+
+# App
+LOG_LEVEL=info
+APP_KEY=
+APP_URL=http://${HOST}:${PORT}
+
+# Session
+SESSION_DRIVER=cookie
+
+#--------------------------------------------------------------------
+# CORS (configure allowed origins for API access)
+#--------------------------------------------------------------------
+# CORS_ORIGIN=http://localhost:5173,http://localhost:3000
+# Newznab API key (/api?apikey=...); leave empty to allow access without a key
+NEWZNAB_API_KEY=

.env.test

@@ -0,0 +1 @@
+SESSION_DRIVER=memory

.gitea/workflows/deploy.yaml

@@ -0,0 +1,81 @@
+# Tests every push; a push to master that passes is deployed to homelab.
+#
+# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
+# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
+# which fetches the commit from this repo, builds the image, runs the
+# migrations and recreates the app services (never postgres or redis).
+name: Test and deploy
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: indexer
+          POSTGRES_PASSWORD: indexer
+          POSTGRES_DB: usenet_indexer
+        options: >-
+          --health-cmd "pg_isready -U indexer -d usenet_indexer"
+          --health-interval 5s
+          --health-retries 20
+      redis:
+        image: redis:8
+    env:
+      TZ: UTC
+      PORT: 3333
+      HOST: 0.0.0.0
+      NODE_ENV: test
+      LOG_LEVEL: warn
+      APP_KEY: ci-only-app-key-not-a-secret-000000
+      APP_URL: http://localhost:3333
+      SESSION_DRIVER: memory
+      DB_CONNECTION: pg
+      DB_HOST: postgres
+      DB_PORT: 5432
+      DB_USER: indexer
+      DB_PASSWORD: indexer
+      DB_DATABASE: usenet_indexer
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      # Tests never connect to a news server.
+      NNTP_HOST: localhost
+      NNTP_PORT: 563
+      NNTP_USER: ci
+      NNTP_PASSWORD: ci
+      NNTP_SECURE: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Type check
+        run: npx tsc --noEmit
+      - name: Migrate
+        run: node ace migration:run --force
+      - name: Test
+        run: node ace test
+
+  deploy:
+    needs: test
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy to homelab
+        env:
+          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+          DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
+        run: |
+          install -m 700 -d ~/.ssh
+          printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
+          ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
+            daan@10.206.0.175 "deploy ${{ github.sha }}"

.gitignore

@@ -1,113 +1,26 @@
-# Logs
+# Dependencies and AdonisJS build
-logs
+node_modules
-*.log
+build
-npm-debug.log*
+tmp/*
-yarn-debug.log*
+!tmp/.gitkeep
-yarn-error.log*
-lerna-debug.log*
 
-# Diagnostic reports (https://nodejs.org/api/report.html)
+# Secrets
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-temporary-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Microbundle cache
-.rpt2_cache/
-.rts2_cache_cjs/
-.rts2_cache_es/
-.rts2_cache_umd/
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
 .env
-.env.test
+.env.local
+.env.production.local
+.env.development.local
 
-# parcel-bundler cache (https://parceljs.org/)
+# Frontend assets compiled code
-.cache
+public/assets
 
-# Next.js build output
+# Build tools specific
-.next
+npm-debug.log
+yarn-error.log
 
-# Nuxt.js build output
+# Editors specific
-.nuxt
+.fleet
+.idea
+.vscode
 
-# Gatsby files
+# Platform specific
-.cache/
+.DS_Store
-# Comment in the public line in if your project uses Gatsby and not Next.js
-# https://nextjs.org/blog/next-9-1#public-directory-support
-# public
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-# DynamoDB Local files
-.dynamodb/
-
-# TernJS port file
-.tern-port
-
-# Stores VSCode versions used for testing VSCode extensions
-.vscode-test
-
-# IDE files
-.idea/
-.vscode/
-*.swp
-database.sqlite
-*.par2
-*.nzb
-files
-*.bin

.nvmrc

@@ -0,0 +1 @@
+24.16

.prettierignore

@@ -0,0 +1,3 @@
+.adonisjs
+node_modules
+build

Dockerfile

@@ -0,0 +1,29 @@
+# syntax=docker/dockerfile:1
+
+# Node is pinned: on 24.21, jsonschema 1.5.0 (used by Ace to validate commands)
+# throws "Invalid URL" on $ref, so no ace command starts. See LOG.md.
+ARG NODE_VERSION=24.16
+
+# Build stage: dev dependencies, TypeScript compile, native addons.
+# The full node image ships python3/make/g++, needed to compile yencode.
+FROM node:${NODE_VERSION}-bookworm AS build
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN npm ci
+COPY . .
+RUN node ace build \
+ && cd build \
+ && npm ci --omit=dev
+
+# Runtime stage: compiled app with production dependencies only.
+FROM node:${NODE_VERSION}-bookworm-slim
+ENV NODE_ENV=production
+# bsdtar (libarchive) unpacks compressed RAR entries, e.g. NZBs posted inside RARs.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends libarchive-tools \
+ && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY --from=build /app/build ./
+USER node
+# Every service picks its own command, e.g. `node ace worker:fetch`.
+CMD ["node", "ace", "list"]

ace.js

@@ -0,0 +1,27 @@
+/*
+|--------------------------------------------------------------------------
+| JavaScript entrypoint for running ace commands
+|--------------------------------------------------------------------------
+|
+| DO NOT MODIFY THIS FILE AS IT WILL BE OVERRIDDEN DURING THE BUILD
+| PROCESS.
+|
+| See docs.adonisjs.com/guides/typescript-build-process#creating-production-build
+|
+| Since, we cannot run TypeScript source code using "node" binary, we need
+| a JavaScript entrypoint to run ace commands.
+|
+| This file registers the "ts-node/esm" hook with the Node.js module system
+| and then imports the "bin/console.ts" file.
+|
+*/
+
+/**
+ * Register hook to process TypeScript files using @poppinss/ts-exec
+ */
+import '@poppinss/ts-exec'
+
+/**
+ * Import ace console entrypoint
+ */
+await import('./bin/console.js')

adonisrc.ts

@@ -0,0 +1,120 @@
+import { indexEntities } from '@adonisjs/core'
+import { defineConfig } from '@adonisjs/core/app'
+import { generateRegistry } from '@tuyau/core/hooks'
+
+export default defineConfig({
+  /*
+  |--------------------------------------------------------------------------
+  | Experimental flags
+  |--------------------------------------------------------------------------
+  |
+  | The following features will be enabled by default in the next major release
+  | of AdonisJS. You can opt into them today to avoid any breaking changes
+  | during upgrade.
+  |
+  */
+  experimental: {},
+
+  /*
+  |--------------------------------------------------------------------------
+  | Commands
+  |--------------------------------------------------------------------------
+  |
+  | List of ace commands to register from packages. The application commands
+  | will be scanned automatically from the "./commands" directory.
+  |
+  */
+  commands: [
+    () => import('@adonisjs/core/commands'),
+    () => import('@adonisjs/lucid/commands'),
+    () => import('@adonisjs/session/commands'),
+  ],
+
+  /*
+  |--------------------------------------------------------------------------
+  | Service providers
+  |--------------------------------------------------------------------------
+  |
+  | List of service providers to import and register when booting the
+  | application
+  |
+  */
+  providers: [
+    () => import('@adonisjs/core/providers/app_provider'),
+    () => import('@adonisjs/core/providers/hash_provider'),
+    {
+      file: () => import('@adonisjs/core/providers/repl_provider'),
+      environment: ['repl', 'test'],
+    },
+    () => import('@adonisjs/core/providers/vinejs_provider'),
+    () => import('@adonisjs/session/session_provider'),
+    () => import('@adonisjs/shield/shield_provider'),
+    () => import('@adonisjs/lucid/database_provider'),
+    () => import('@adonisjs/cors/cors_provider'),
+    () => import('@adonisjs/auth/auth_provider'),
+    () => import('#providers/api_provider'),
+    () => import('@adonisjs/static/static_provider')
+  ],
+
+  /*
+  |--------------------------------------------------------------------------
+  | Preloads
+  |--------------------------------------------------------------------------
+  |
+  | List of modules to import before starting the application.
+  |
+  */
+  preloads: [
+    () => import('#start/routes'),
+    () => import('#start/kernel'),
+    () => import('#start/validator'),
+  ],
+
+  /*
+  |--------------------------------------------------------------------------
+  | Tests
+  |--------------------------------------------------------------------------
+  |
+  | List of test suites to organize tests by their type. Feel free to remove
+  | and add additional suites.
+  |
+  */
+  tests: {
+    suites: [
+      {
+        files: ['tests/unit/**/*.spec.{ts,js}'],
+        name: 'unit',
+        timeout: 2000,
+      },
+      {
+        files: ['tests/functional/**/*.spec.{ts,js}'],
+        name: 'functional',
+        timeout: 30000,
+      },
+    ],
+    forceExit: false,
+  },
+
+  /*
+  |--------------------------------------------------------------------------
+  | Metafiles
+  |--------------------------------------------------------------------------
+  |
+  | A collection of files you want to copy to the build folder when creating
+  | the production build.
+  |
+  */
+  metaFiles: [{
+    pattern: 'public/**',
+    reloadServer: false,
+  }],
+
+  hooks: {
+    init: [
+      indexEntities({
+        transformers: { enabled: true },
+      }),
+      generateRegistry(),
+    ],
+  },
+})

app/controllers/access_tokens_controller.ts

@@ -0,0 +1,29 @@
+import User from '#models/user'
+import { loginValidator } from '#validators/user'
+import type { HttpContext } from '@adonisjs/core/http'
+import UserTransformer from '#transformers/user_transformer'
+
+export default class AccessTokensController {
+  async store({ request, serialize }: HttpContext) {
+    const { email, password } = await request.validateUsing(loginValidator)
+
+    const user = await User.verifyCredentials(email, password)
+    const token = await User.accessTokens.create(user)
+
+    return serialize({
+      user: UserTransformer.transform(user),
+      token: token.value!.release(),
+    })
+  }
+
+  async destroy({ auth }: HttpContext) {
+    const user = auth.getUserOrFail()
+    if (user.currentAccessToken) {
+      await User.accessTokens.delete(user, user.currentAccessToken.identifier)
+    }
+
+    return {
+      message: 'Logged out successfully',
+    }
+  }
+}

app/controllers/new_account_controller.ts

@@ -0,0 +1,18 @@
+import User from '#models/user'
+import { signupValidator } from '#validators/user'
+import type { HttpContext } from '@adonisjs/core/http'
+import UserTransformer from '#transformers/user_transformer'
+
+export default class NewAccountController {
+  async store({ request, serialize }: HttpContext) {
+    const { fullName, email, password } = await request.validateUsing(signupValidator)
+
+    const user = await User.create({ fullName, email, password })
+    const token = await User.accessTokens.create(user)
+
+    return serialize({
+      user: UserTransformer.transform(user),
+      token: token.value!.release(),
+    })
+  }
+}

app/controllers/newznab_controller.ts

@@ -0,0 +1,123 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import env from '#start/env'
+import SearchService from '#services/SearchService'
+import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
+
+const LIMITS = { max: 100, default: 50 }
+
+/** Search functions and the category each defaults to when `cat` is absent. */
+const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
+  'search': undefined,
+  'tvsearch': 5000,
+  'tv-search': 5000,
+  'movie': 2000,
+  'movie-search': 2000,
+  'music': 3000,
+  'audio': 3000,
+  'book': 7000,
+}
+
+/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */
+function configuredApiKey(): string | undefined {
+  const value: any = env.get('NEWZNAB_API_KEY')
+  const key = typeof value === 'string' ? value : value?.release()
+  return key || undefined
+}
+
+function int(value: unknown): number | undefined {
+  return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined
+}
+
+/**
+ * Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like:
+ * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
+ */
+export default class NewznabController {
+  async handle({ request, response }: HttpContext) {
+    const qs = request.qs()
+    const t = String(qs.t ?? '')
+    const server = this.server(request)
+
+    const xml = (body: string, type = 'application/xml') => {
+      response.header('Content-Type', `${type}; charset=utf-8`)
+      return response.send(body)
+    }
+
+    const apiKey = configuredApiKey()
+    if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
+
+    if (t === 'caps') return xml(capsXml(server, LIMITS))
+
+    if (t === 'get') {
+      const id = int(qs.id)
+      if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
+      const nzb = await SearchService.nzb(id)
+      if (!nzb) return xml(errorXml(300, 'No such item'))
+      const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
+      response.header('Content-Disposition', `attachment; filename="${filename}"`)
+      return xml(nzb.xml, 'application/x-nzb')
+    }
+
+    if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
+
+    const requested = String(qs.cat ?? '')
+      .split(',')
+      .map((c) => int(c.trim()))
+      .filter((c): c is number => c !== undefined)
+    const fallback = SEARCH_FUNCTIONS[t]
+    const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : [])
+    const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
+    const offset = int(qs.offset) ?? 0
+
+    // ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but
+    // answer with an empty result rather than everything.
+    const q = typeof qs.q === 'string' ? qs.q.trim() : ''
+    const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
+
+    const result = idOnly
+      ? { total: 0, releases: [] }
+      : await SearchService.search({
+          q,
+          categories,
+          readable: true,
+          notIncomplete: true,
+          hideContainers: true,
+          maxAgeDays: int(qs.maxage),
+          season: int(qs.season),
+          episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
+          sort: 'newest',
+          perPage: limit,
+          offset,
+        })
+
+    const items = result.releases.map((r: any) => {
+      const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
+      return {
+        id: r.id,
+        title: r.name,
+        category: r.category ?? 8010,
+        size: r.size,
+        files: r.fileCount,
+        postedAt: r.postedAt ? new Date(r.postedAt) : null,
+        poster: r.poster,
+        groups: r.groups ?? [],
+        // 1 only when a password is needed that we can't give; a known one is in our NZB.
+        password: r.passwordState === 'missing',
+        description: r.spotDescription ?? undefined,
+        season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
+        episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
+      }
+    })
+    return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml')
+  }
+
+  private server(request: HttpContext['request']): NewznabServer {
+    const proto = request.header('x-forwarded-proto') ?? 'http'
+    const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
+    return {
+      title: 'Usenet Indexer',
+      baseUrl: `${proto}://${host}`,
+      apiKey: configuredApiKey(),
+    }
+  }
+}

app/controllers/profile_controller.ts

@@ -0,0 +1,8 @@
+import UserTransformer from '#transformers/user_transformer'
+import type { HttpContext } from '@adonisjs/core/http'
+
+export default class ProfileController {
+  async show({ auth, serialize }: HttpContext) {
+    return serialize(UserTransformer.transform(auth.getUserOrFail()))
+  }
+}

app/controllers/releases_controller.ts

@@ -0,0 +1,56 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import SearchService, { type SortOrder } from '#services/SearchService'
+
+const flag = (value: unknown) => value === '1' || value === 'true' || value === true
+
+export default class ReleasesController {
+  /** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&source=&poster=&sort=&page=&per_page= */
+  async index({ request }: HttpContext) {
+    const qs = request.qs()
+    return SearchService.search({
+      q: typeof qs.q === 'string' ? qs.q : undefined,
+      readable: flag(qs.readable),
+      complete: flag(qs.complete),
+      hideEncrypted: flag(qs.hide_encrypted),
+      includePar2Only: flag(qs.par2_only),
+      source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
+      poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
+      sort: qs.sort as SortOrder | undefined,
+      page: Number(qs.page) || 1,
+      perPage: Number(qs.per_page) || 50,
+      // The page shows "1,000+"; counting further made filter clicks take seconds.
+      countLimit: 1000,
+    })
+  }
+
+  /** GET /api/v1/releases/:id */
+  async show({ params, response }: HttpContext) {
+    const release = await SearchService.release(Number(params.id))
+    return release ?? response.notFound({ error: 'Release not found' })
+  }
+
+  /** GET /api/v1/releases/:id/nzb */
+  async nzb({ params, response }: HttpContext) {
+    const nzb = await SearchService.nzb(Number(params.id))
+    if (!nzb) return response.notFound({ error: 'Release not found' })
+    const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
+    response.header('Content-Type', 'application/x-nzb')
+    response.header('Content-Disposition', `attachment; filename="${filename}"`)
+    return response.send(nzb.xml)
+  }
+
+  /** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */
+  async image({ params, response }: HttpContext) {
+    const image = await SearchService.spotImage(Number(params.id))
+    if (!image) return response.notFound({ error: 'No image' })
+    response.header('Content-Type', image.type)
+    // A spot's image never changes.
+    response.header('Cache-Control', 'public, max-age=604800, immutable')
+    return response.send(image.data)
+  }
+
+  /** GET /api/v1/stats */
+  async stats() {
+    return SearchService.stats()
+  }
+}

app/exceptions/handler.ts

@@ -0,0 +1,28 @@
+import app from '@adonisjs/core/services/app'
+import { type HttpContext, ExceptionHandler } from '@adonisjs/core/http'
+
+export default class HttpExceptionHandler extends ExceptionHandler {
+  /**
+   * In debug mode, the exception handler will display verbose errors
+   * with pretty printed stack traces.
+   */
+  protected debug = !app.inProduction
+
+  /**
+   * The method is used for handling errors and returning
+   * response to the client
+   */
+  async handle(error: unknown, ctx: HttpContext) {
+    return super.handle(error, ctx)
+  }
+
+  /**
+   * The method is used to report error to the logging service or
+   * the a third party error monitoring service.
+   *
+   * @note You should not attempt to send a response from this method.
+   */
+  async report(error: unknown, ctx: HttpContext) {
+    return super.report(error, ctx)
+  }
+}

app/middleware/auth_middleware.ts

@@ -0,0 +1,20 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import type { NextFn } from '@adonisjs/core/types/http'
+import type { Authenticators } from '@adonisjs/auth/types'
+
+/**
+ * Auth middleware is used authenticate HTTP requests and deny
+ * access to unauthenticated users.
+ */
+export default class AuthMiddleware {
+  async handle(
+    ctx: HttpContext,
+    next: NextFn,
+    options: {
+      guards?: (keyof Authenticators)[]
+    } = {}
+  ) {
+    await ctx.auth.authenticateUsing(options.guards)
+    return next()
+  }
+}

app/middleware/container_bindings_middleware.ts

@@ -0,0 +1,19 @@
+import { Logger } from '@adonisjs/core/logger'
+import { HttpContext } from '@adonisjs/core/http'
+import { type NextFn } from '@adonisjs/core/types/http'
+
+/**
+ * The container bindings middleware binds classes to their request
+ * specific value using the container resolver.
+ *
+ * - We bind "HttpContext" class to the "ctx" object
+ * - And bind "Logger" class to the "ctx.logger" object
+ */
+export default class ContainerBindingsMiddleware {
+  handle(ctx: HttpContext, next: NextFn) {
+    ctx.containerResolver.bindValue(HttpContext, ctx)
+    ctx.containerResolver.bindValue(Logger, ctx.logger)
+
+    return next()
+  }
+}

app/middleware/force_json_response_middleware.ts

@@ -0,0 +1,9 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import type { NextFn } from '@adonisjs/core/types/http'
+
+export default class ForceJsonResponseMiddleware {
+  handle(ctx: HttpContext, next: NextFn) {
+    ctx.request.request.headers.accept = 'application/json'
+    return next()
+  }
+}

app/middleware/silent_auth_middleware.ts

@@ -0,0 +1,16 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import type { NextFn } from '@adonisjs/core/types/http'
+
+/**
+ * Silent auth middleware can be used as a global middleware to silent check
+ * if the user is logged-in or not.
+ *
+ * The request continues as usual, even when the user is not logged-in.
+ */
+export default class SilentAuthMiddleware {
+  async handle(ctx: HttpContext, next: NextFn) {
+    await ctx.auth.check()
+
+    return next()
+  }
+}

app/models/base_model.ts

@@ -0,0 +1,4 @@
+import { BaseModel as AdonisBaseModel } from '@adonisjs/lucid/orm'
+
+export default class BaseModel extends AdonisBaseModel {
+}

app/models/file.ts

@@ -0,0 +1,41 @@
+import { DateTime } from 'luxon'
+import { column } from '@adonisjs/lucid/orm'
+import BaseModel from '#models/base_model'
+import { loadedSegments, storedSegments } from '#services/FileStore'
+import type { Segment } from '#lib/nzb/index'
+
+export default class File extends BaseModel {
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column({ columnName: 'file_key' })
+  declare fileKey: string | null
+
+  @column()
+  declare filename: string
+
+  @column()
+  declare poster: string
+
+  @column()
+  declare date: number
+
+  @column()
+  declare parts: number
+
+  /** Stored compressed (see FileStore); the model sees the list. */
+  @column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) })
+  declare segments: Segment[]
+
+  @column({ columnName: 'first_message_id' })
+  declare firstMessageId: string | null
+
+  @column()
+  declare groups: string[]
+
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime
+}

app/models/group.ts

@@ -0,0 +1,32 @@
+import { DateTime } from 'luxon'
+import { column } from '@adonisjs/lucid/orm'
+import BaseModel from '#models/base_model'
+
+export default class Group extends BaseModel {
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare name: string
+
+  @column()
+  declare active: boolean
+
+  /** Every article up to here was fetched successfully (see FetchRangeService). */
+  @column({ columnName: 'last_indexed_id' })
+  declare lastIndexedId: bigint
+
+  /** Download bodies of obfuscated articles to identify them (bandwidth-heavy). */
+  @column({ columnName: 'fetch_bodies' })
+  declare fetchBodies: boolean
+
+  /** How far the scheduler has queued fetch jobs. */
+  @column({ columnName: 'last_queued_id' })
+  declare lastQueuedId: bigint
+
+  @column.dateTime({ autoCreate: true })
+  declare createdAt: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updatedAt: DateTime
+}

app/models/user.ts

@@ -0,0 +1,18 @@
+import { UserSchema } from '#database/schema'
+import hash from '@adonisjs/core/services/hash'
+import { compose } from '@adonisjs/core/helpers'
+import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
+import { type AccessToken, DbAccessTokensProvider } from '@adonisjs/auth/access_tokens'
+
+export default class User extends compose(UserSchema, withAuthFinder(hash)) {
+  static accessTokens = DbAccessTokensProvider.forModel(User)
+  declare currentAccessToken?: AccessToken
+
+  get initials() {
+    const [first, last] = this.fullName ? this.fullName.split(' ') : this.email.split('@')
+    if (first && last) {
+      return `${first.charAt(0)}${last.charAt(0)}`.toUpperCase()
+    }
+    return `${first.slice(0, 2)}`.toUpperCase()
+  }
+}

app/services/ArchiveService.ts

@@ -0,0 +1,178 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import IngestService from '#services/IngestService'
+import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
+import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
+
+/** Stored per file and release; a listing of thousands of files is not useful. */
+const MAX_ENTRIES = 500
+
+export interface ArchiveResult {
+  info: RarInfo
+  releaseId?: number
+  releaseRenamed?: string
+  grouped?: boolean
+}
+
+/**
+ * Records what a RAR volume's headers say: the listing on the file, the
+ * release it belongs to (from the shared directory or continued entry, when
+ * the file has none yet), and for first volumes the contents and password
+ * flag on the release. See CONCEPT.md (D19, D21).
+ */
+class ArchiveService {
+  async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
+    const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
+    await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) })
+
+    let releaseId = file.release_id ?? binary?.release_id ?? null
+    let grouped = false
+    const identity = rarReleaseIdentity(info.entries)
+    if (!releaseId && identity) {
+      const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
+        {
+          release: identity,
+          poster: binary?.poster ?? file.poster,
+          weakIdentity: true,
+          groups: binary?.groups ?? file.groups ?? [],
+          postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
+        },
+      ])
+      releaseId = ids.get(identity.key)!
+      grouped = true
+      if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
+      await db.from('files').where('id', file.id).update({ release_id: releaseId })
+    }
+    if (!releaseId) return { info }
+
+    const release = await db.from('releases').where('id', releaseId).first()
+    const update: Record<string, unknown> = {}
+    if (info.firstVolume !== false) {
+      update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
+      if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
+    } else if (info.headersEncrypted) {
+      update.encrypted = true
+    }
+    const better = releaseNameFromContents(info)
+    // A spot's title is kept (CONCEPT.md D29).
+    const renamed =
+      release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
+    if (renamed) update.name = renamed
+    if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
+
+    if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
+    return { info, releaseId, releaseRenamed: renamed, grouped }
+  }
+}
+
+const run = promisify(execFile)
+
+/** Tools that print one archive entry to stdout, tried in order. */
+const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
+  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
+  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
+]
+
+/**
+ * The unpacked data of one entry of a complete (single-volume) archive:
+ * copied out when stored, else unpacked with bsdtar (libarchive, in the
+ * Docker image) or unrar. Null when that is not possible.
+ */
+export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
+  const stored = storedEntryData(archive, entry)
+  if (stored) return stored
+  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
+  if (entry.size !== null && entry.size > maxBytes) return null
+
+  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
+  try {
+    const file = path.join(dir, 'archive.rar')
+    await writeFile(file, archive)
+    for (const [tool, args] of EXTRACTORS) {
+      try {
+        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
+        if (stdout.length > 0) return stdout
+      } catch (error: any) {
+        if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
+      }
+    }
+    return null
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}
+
+/**
+ * A release name from an archive listing: the top-level directory shared by
+ * all entries when it reads like a release name, otherwise the stem of the
+ * largest entry with a readable name (e.g. an archive holding just
+ * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
+ */
+export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
+  const entries = info.entries.filter((e) => !e.directory)
+  const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0]))
+  if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) {
+    const [top] = tops
+    if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
+  }
+  const candidates = entries
+    .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
+    .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
+    .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
+    .sort((a, b) => b.size - a.size)
+  return candidates[0]?.stem
+}
+
+/**
+ * Renames random-named releases from their stored archive listing (no
+ * downloads). Used by `release:rename`.
+ */
+export async function renameFromStoredContents(): Promise<number> {
+  const releases = await db
+    .from('releases')
+    .whereNotNull('contents')
+    .whereRaw("jsonb_array_length(contents) > 0")
+    .select('id', 'name', 'source', 'contents')
+  let renamed = 0
+  for (const release of releases) {
+    if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
+    const better = releaseNameFromContents({ entries: release.contents })
+    if (!better || better === release.name) continue
+    await db.from('releases').where('id', release.id).update({ name: better })
+    renamed++
+  }
+  return renamed
+}
+
+/**
+ * Obfuscated names: one long token of mixed letters and digits, no
+ * separators, and the other generated shapes seen on Usenet (LOG.md
+ * 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
+ */
+export function looksRandom(name: string): boolean {
+  // An unknown short extension (e.g. ".bin") says nothing about the name.
+  name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
+  // VIDEO_TS, Sample, CD1: real words, but no name.
+  if (isGenericName(name)) return true
+  if (/^[0-9a-f]{16,}$/i.test(name)) return true
+  // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
+  if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
+  // UUIDs
+  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
+  // 1650777167 (yEncBin), "83111763 n" (rclone uploads)
+  if (/^\d{6,}( n)?$/.test(name)) return true
+  // prvjr7hapd0i1ny14pz4, and rclone's long base32 names
+  if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
+  // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
+  if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
+  if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
+  // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
+  return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
+}
+
+export default new ArchiveService()

app/services/BackfillService.ts

@@ -0,0 +1,104 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import { articleAtDate } from '#lib/nntp/date_search'
+
+/** Articles XOVERed to find a posting date near an article number. */
+const PROBE = 200n
+
+/**
+ * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
+ * where live indexing began back to the article posted N days ago; `tick`
+ * (every scheduler run) queues low-priority, header-only fetch ranges from
+ * that cursor downwards, newest first, while few are waiting and the
+ * database stays within its size limits.
+ */
+class BackfillService {
+  /** Sets the group's backfill to reach back `days` days. Returns its range. */
+  async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
+    const group = await db.from('groups').where('name', groupName).first()
+    if (!group) throw new Error(`Unknown group ${groupName}`)
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
+      const dateOf = async (n: bigint) => {
+        const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
+        const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
+        const date = header ? Date.parse(header.date) : Number.NaN
+        return Number.isNaN(date) ? null : date
+      }
+      const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
+      // Live indexing began a little before the group was added: continue from there.
+      const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
+      const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
+      await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
+      return next > until ? { from: until, to: next } : null
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+
+  /** Stops a group's backfill (queued ranges still run). */
+  async stop(groupName: string) {
+    await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
+  }
+
+  /** Queues the next backfill ranges. Returns how many. */
+  async tick(): Promise<number> {
+    const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
+    const groups = await db
+      .from('groups')
+      .where('active', true)
+      .whereNotNull('backfill_until_id')
+      .whereRaw('backfill_next_id > backfill_until_id')
+      .orderBy('id')
+    if (groups.length === 0) return 0
+
+    const room = await this.room(maxDefaultGb, maxHddGb)
+    if (room !== true) {
+      logger.warn(`Backfill paused: ${room}.`)
+      return 0
+    }
+    const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
+    let free = maxQueued - Number(count)
+    let queued = 0
+    // Round-robin over the groups, one range each per pass.
+    while (free > 0 && groups.length > 0) {
+      for (const group of [...groups]) {
+        if (free <= 0) break
+        const next = BigInt(group.backfill_next_id)
+        const until = BigInt(group.backfill_until_id)
+        const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
+        await FetchRangeService.queue(group, start, next, { backfill: true })
+        group.backfill_next_id = (start - 1n).toString()
+        await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
+        free--
+        queued++
+        if (start - 1n <= until) {
+          groups.splice(groups.indexOf(group), 1)
+          logger.info(`${group.name}: backfill fully queued.`)
+        }
+      }
+    }
+    return queued
+  }
+
+  /** True when both parts of the database are under their limits, else why not. */
+  private async room(maxDefaultGb: number, maxHddGb: number): Promise<true | string> {
+    const { rows } = await db.rawQuery(
+      `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
+    )
+    const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
+    if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
+    if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
+    return true
+  }
+}
+
+export default new BackfillService()

app/services/ContentService.ts

@@ -0,0 +1,280 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import Par2Service, { type Par2Result } from '#services/Par2Service'
+import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
+import NameService from '#services/NameService'
+import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
+import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
+import {
+  isArchiveStart,
+  NICOVIDEO_FILE,
+  NICOVIDEO_RELEASE,
+  nicovideoInfo,
+  nicovideoReleaseName,
+  sniffContentType,
+  zstdPrefix,
+  type ContentType,
+  type NicovideoInfo,
+} from '#lib/content/index'
+import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
+import { fileStem, releaseIdentity } from '#lib/release/index'
+import IngestService from '#services/IngestService'
+import { looksLikeFilename } from '#lib/subject/index'
+import { sevenZipVolume } from '#lib/sevenzip/index'
+import SevenZipService from '#services/SevenZipService'
+import SfvService from '#services/SfvService'
+import { segmentsOf } from '#services/FileStore'
+
+const FIRST_CHUNK = 16 * 1024
+
+/**
+ * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
+ * the file's first part. Undefined when that part is shorter than 16 KiB but
+ * the file is not (the chunk would span two parts).
+ */
+export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
+  return chunkMd5(decodeSegment(body).data, fileSize)
+}
+
+function chunkMd5(data: Buffer, fileSize: number): string | undefined {
+  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
+  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
+}
+
+export interface ContentResult {
+  par2?: Par2Result
+  archive?: ArchiveResult
+  named: number
+  contentType: ContentType | null
+  nzb?: NzbImportResult
+  /** NZBs read from inside an archive. */
+  archiveNzbs?: NzbImportResult[]
+  /** Files an SFV's listing moved into its release. */
+  sfvMoved?: number
+  /** A niconico capture's release name from its API response (D34). */
+  nicovideo?: string
+  /** The real name from the yEnc header, for a file posted under a random subject (D36). */
+  yencName?: string
+}
+
+/**
+ * Takes names and groupings from file contents (CONCEPT.md D21). For a
+ * completed file with a random name, one article (the first) is enough:
+ * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
+ * - par2 files are recognised by content and name every file of their set,
+ * - RAR headers give the release grouping, name and contents.
+ */
+class ContentService {
+  /** Random or missing names, and named first RAR volumes (for contents and password flag). */
+  isCandidate(name: string) {
+    if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
+    if (isArchiveStart(name)) return true
+    if (/\.nzb$/i.test(name)) return true
+    // SFVs name the files of their release; niconico captures carry their title (D34).
+    if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
+    // Every 7z volume: the set is listed when its last volume is in (see inspect).
+    if (sevenZipVolume(name) !== null) return true
+    return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
+  }
+
+  /**
+   * Uploads that randomise the subject per article often keep the real file
+   * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
+   * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
+   * leaves its weak one-file release for the release its name gives, where
+   * the other files of the upload gather. Returns the name, or undefined.
+   */
+  private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
+    const current = String(file.real_name ?? file.filename)
+    if (file.real_name || !looksRandom(fileStem(current))) return undefined
+    if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
+
+    const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
+    const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
+    let releaseId = file.release_id
+    if (weak) {
+      const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
+      if (identity) {
+        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
+          { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
+        ])
+        releaseId = ids.get(identity.key) ?? releaseId
+      }
+    }
+    await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
+    if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
+    return yencName
+  }
+
+  /**
+   * Names a niconico capture's release after the video, while it still has
+   * the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
+   * the search text.
+   */
+  private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
+    if (!file.release_id) return undefined
+    const release = await db.from('releases').where('id', file.release_id).first()
+    if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
+    const name = nicovideoReleaseName(info, id)
+    await db
+      .from('releases')
+      .where('id', release.id)
+      .update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
+    return name
+  }
+
+  /** Downloads a whole (small) file whose first segment is already decoded. */
+  private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
+    const parts = segmentsOf(file)
+    const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
+    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
+    if (parts.length === 1) return first
+
+    let assembled: YencFile | undefined
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      for (const part of parts) {
+        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
+        assembled ??= YencFile.fromSegment(segment)
+        assembled.add(segment)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    return assembled!.getBuffer()
+  }
+
+  /**
+   * Downloads a small archive that lists NZBs and imports each of them
+   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
+   * uploads by other posters.
+   */
+  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
+    const archive = await this.readWhole(first, file)
+    const info = archive && parseRarHeaders(archive)
+    if (!archive || !info) return []
+    const entries = info.entries.filter(isNzbEntry)
+    const results: NzbImportResult[] = []
+    for (const entry of entries) {
+      const xml = await extractEntry(archive, entry)
+      if (!xml) continue
+      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
+        entryName: entry.name,
+        moveContainer: entries.length === 1,
+      })
+      if (imported) results.push(imported)
+    }
+    return results
+  }
+
+  async inspect(fileId: number): Promise<ContentResult | null> {
+    const file = await db.from('files').where('id', fileId).first()
+    const binary = await db.from('binaries').where('file_id', fileId).first()
+    if (!file || !this.isCandidate(file.filename)) return null
+    // A later 7z volume: list the set from its first volume once all volumes are in.
+    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
+      const volumes = await SevenZipService.volumes(file)
+      if (!volumes || volumes[0].archive) return null
+      return this.inspect(Number(volumes[0].id))
+    }
+    const first = segmentsOf(file)[0]
+    if (!first?.id) return null
+
+    const conn = await NntpService.acquire()
+    let failed = false
+    let body: Buffer
+    try {
+      body = await NntpService.fetchBody(conn, first.id)
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
+    let segment
+    try {
+      segment = decodeSegment(body)
+    } catch (error) {
+      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+      segment = decodeSegment(body, { lenientSize: true })
+    }
+    const size = segment.meta.begin.size
+    const contentType = sniffContentType(segment.data)
+    await db.from('files').where('id', fileId).update({ content_type: contentType })
+
+    // Fingerprint, so par2 sets (now or later) can name this file exactly.
+    if (binary) {
+      await db
+        .from('binaries')
+        .where('id', binary.id)
+        .update({
+          size: db.raw('coalesce(size, ?)', [size]),
+          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
+        })
+    }
+
+    const result: ContentResult = { named: 0, contentType }
+    result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
+    if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
+      // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
+      // to read, and useless without the key.
+      const releaseId = file.release_id ?? binary?.release_id
+      if (releaseId) {
+        await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
+      }
+      return result
+    }
+    const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
+    if (nicovideo) {
+      const info = nicovideoInfo(await zstdPrefix(segment.data))
+      if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
+      return result
+    }
+    if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
+      const text = await this.readWhole(segment.data, file)
+      if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
+      return result
+    }
+    if (contentType === 'nzb') {
+      const xml = await this.readWhole(segment.data, file)
+      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
+      return result
+    }
+    const set = await Par2Service.read(segment.data, segmentsOf(file))
+    if (set && binary) {
+      result.par2 = await Par2Service.attach(binary, set)
+    } else if (contentType === '7z') {
+      const info = await SevenZipService.list(file, segment.data)
+      if (info) result.archive = await ArchiveService.record(file, binary, info)
+    } else {
+      const info = parseRarHeaders(segment.data)
+      if (info) {
+        result.archive = await ArchiveService.record(file, binary, info)
+        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
+      }
+    }
+    if (binary) result.named = await NameService.resolve([binary.id])
+    return result
+  }
+}
+
+/** NZBs that can be read from a single, unencrypted archive. */
+function hasNzbEntries(info: RarInfo): boolean {
+  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+}
+
+function isNzbEntry(entry: RarEntry): boolean {
+  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+}
+
+/** Small posted files (NZBs) that are read completely. */
+const WHOLE_MAX_PARTS = 40
+const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+
+export default new ContentService()

app/services/DeferredService.ts

@@ -0,0 +1,264 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import { type OverviewHeader } from '#services/IngestService'
+import { decodeSegment, inferTotalParts, parseYencMeta } from '#lib/yenc/index'
+import { sniffContentType } from '#lib/content/index'
+import { parseRarHeaders } from '#lib/rar/index'
+
+/** Expanding re-reads the file's article range with XOVER; beyond this it is too costly. */
+const EXPAND_MAX_RANGE = 200_000
+/** A claimed row whose worker died is taken again after this long. */
+const RECLAIM_AFTER = '1 hour'
+
+export interface DeferredArticle {
+  articleNumber: number
+  header: OverviewHeader
+}
+
+/** File name from a deferred uploader's subject: `[i/M] "name" yEnc`, no part counter. */
+export function deferredName(subject: string | undefined): string | null {
+  return /"([^"]+)"/.exec(subject ?? '')?.[1]?.trim() || null
+}
+
+export function isDeferred(header: OverviewHeader): boolean {
+  const id = header['message-id'] ?? ''
+  return indexerConfig.deferBodyMessageIds.some((re) => re.test(id))
+}
+
+/** A recorded range never spans more articles than this (one XOVER to expand it). */
+const RANGE_MAX_SPAN = 50_000
+
+export function bodyJob(header: OverviewHeader, group: string, articleNumber: number | bigint) {
+  return {
+    name: 'process-body',
+    data: { header, group, articleNumber: Number(articleNumber) },
+    // By message-id: an article crossposted to several indexed groups is downloaded once.
+    opts: { jobId: `body-${createHash('sha1').update(header['message-id'] ?? '').digest('hex')}` },
+  }
+}
+
+/**
+ * Deferred uploads (CONCEPT.md D28). Their articles skip the body queue and
+ * are recorded here per file, from the headers alone. `worker:deferred`
+ * later downloads one body per file for its size, part count and content
+ * type, and sends small files (NZBs in RARs, par2 index files) to the body
+ * queue completely.
+ */
+class DeferredService {
+  /**
+   * Records one group's deferred articles: per file when the subject names
+   * the file, otherwise only as article ranges. Returns the number of files.
+   */
+  async record(groupName: string, articles: DeferredArticle[]): Promise<number> {
+    const named = articles.filter((a) => deferredName(a.header.subject) !== null)
+    await this.recordRanges(groupName, articles.filter((a) => deferredName(a.header.subject) === null))
+    return this.recordFiles(groupName, named)
+  }
+
+  private async recordRanges(groupName: string, articles: DeferredArticle[]) {
+    if (articles.length === 0) return
+    const sorted = [...articles].sort((a, b) => Number(a.articleNumber) - Number(b.articleNumber))
+    const now = new Date()
+    const rows: Record<string, unknown>[] = []
+    for (const article of sorted) {
+      const n = Number(article.articleNumber)
+      const bytes = Number.parseInt(article.header[':bytes'] ?? '0', 10) || 0
+      const last = rows.at(-1)
+      if (last && n - (last.first_article as number) <= RANGE_MAX_SPAN) {
+        last.last_article = n
+        ;(last.article_count as number)++
+        last.bytes = (last.bytes as number) + bytes
+      } else {
+        rows.push({ group_name: groupName, first_article: n, last_article: n, article_count: 1, bytes, created_at: now })
+      }
+    }
+    await db.table('deferred_ranges').multiInsert(rows)
+  }
+
+  private async recordFiles(groupName: string, articles: DeferredArticle[]): Promise<number> {
+    const files = new Map<string, {
+      name: string; subject: string; poster: string; first: number; firstId: string; last: number
+      count: number; bytes: number; postedAt: Date | null
+    }>()
+    for (const { articleNumber, header } of articles) {
+      const name = deferredName(header.subject)!
+      const n = Number(articleNumber)
+      const bytes = Number.parseInt(header[':bytes'] ?? '0', 10) || 0
+      const date = header.date ? new Date(header.date) : null
+      const postedAt = date && !Number.isNaN(date.getTime()) ? date : null
+      const file = files.get(name)
+      if (!file) {
+        files.set(name, {
+          name, subject: header.subject ?? '', poster: header.from ?? '', first: n,
+          firstId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), last: n, count: 1, bytes, postedAt,
+        })
+        continue
+      }
+      if (n < file.first) {
+        file.first = n
+        file.firstId = (header['message-id'] ?? '').replace(/^<|>$/g, '')
+      }
+      file.last = Math.max(file.last, n)
+      file.count++
+      file.bytes += bytes
+      if (postedAt && (!file.postedAt || postedAt < file.postedAt)) file.postedAt = postedAt
+    }
+    if (files.size === 0) return 0
+
+    const now = new Date()
+    const rows = [...files.values()].map((f) => ({
+      file_key: createHash('sha1').update(`deferred\n${f.name}`).digest('hex'),
+      name: f.name,
+      subject: f.subject,
+      poster: f.poster,
+      group_name: groupName,
+      first_article: f.first,
+      first_message_id: f.firstId,
+      last_article: f.last,
+      article_count: f.count,
+      bytes: f.bytes,
+      posted_at: f.postedAt,
+      first_seen_at: now,
+      last_seen_at: now,
+    }))
+    // Counts and the article range only add up within the group the file was first seen in.
+    await db.rawQuery(
+      `INSERT INTO deferred_files AS d (${Object.keys(rows[0]).join(', ')})
+       SELECT * FROM jsonb_to_recordset(?::jsonb) AS r(
+         file_key varchar, name text, subject text, poster text, group_name text, first_article bigint,
+         first_message_id text, last_article bigint, article_count int, bytes bigint, posted_at timestamptz,
+         first_seen_at timestamptz, last_seen_at timestamptz)
+       ON CONFLICT (file_key) DO UPDATE SET
+         last_seen_at = EXCLUDED.last_seen_at,
+         first_message_id = CASE WHEN d.group_name = EXCLUDED.group_name AND EXCLUDED.first_article < d.first_article
+           THEN EXCLUDED.first_message_id ELSE d.first_message_id END,
+         first_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN LEAST(d.first_article, EXCLUDED.first_article) ELSE d.first_article END,
+         last_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN GREATEST(d.last_article, EXCLUDED.last_article) ELSE d.last_article END,
+         article_count = d.article_count + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.article_count ELSE 0 END,
+         bytes = d.bytes + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.bytes ELSE 0 END,
+         posted_at = LEAST(d.posted_at, EXCLUDED.posted_at)`,
+      [JSON.stringify(rows)]
+    )
+    return files.size
+  }
+
+  /** Takes up to `limit` files that stopped growing; they are marked `processing`. */
+  async claim(limit: number): Promise<any[]> {
+    const settled = new Date(Date.now() - indexerConfig.deferredSettleMs)
+    const { rows } = await db.rawQuery(
+      `UPDATE deferred_files SET status = 'processing', processed_at = now()
+       WHERE id IN (
+         SELECT id FROM deferred_files
+         WHERE (status = 'pending' AND last_seen_at < ?)
+            OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
+         ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED)
+       RETURNING *`,
+      [settled, limit]
+    )
+    return rows
+  }
+
+  /**
+   * Queues the bodies of up to `limit` recorded ranges, oldest first: the
+   * articles are found again with XOVER. Returns ranges and articles queued.
+   */
+  async expandRanges(limit: number): Promise<{ ranges: number; articles: number }> {
+    const ranges = await db.from('deferred_ranges').whereNull('expanded_at').orderBy('id').limit(limit)
+    let articles = 0
+    for (const range of ranges) {
+      const conn = await NntpService.acquire()
+      let failed = false
+      let overviews: [number, OverviewHeader][]
+      try {
+        await conn.group(range.group_name)
+        overviews = (await conn.xover(String(range.first_article), String(range.last_article))).overviews
+      } catch (error) {
+        failed = true
+        throw error
+      } finally {
+        NntpService.release(conn, failed)
+      }
+      const jobs = overviews
+        .filter(([, header]) => header && isDeferred(header) && deferredName(header.subject) === null)
+        .map(([articleNumber, header]) => bodyJob(header, range.group_name, articleNumber))
+      for (let i = 0; i < jobs.length; i += 5000) await QueueService.bodyQueue.addBulk(jobs.slice(i, i + 5000))
+      await db.from('deferred_ranges').where('id', range.id).update({ expanded_at: new Date() })
+      articles += jobs.length
+    }
+    return { ranges: ranges.length, articles }
+  }
+
+  /** Reads one body of the file; expands small files into the body queue. Returns the new status. */
+  async process(file: any): Promise<string> {
+    const update: Record<string, unknown> = { processed_at: new Date(), error: null }
+    try {
+      const conn = await NntpService.acquire()
+      let failed = false
+      let body: Buffer
+      try {
+        body = await NntpService.fetchBody(conn, file.first_message_id)
+      } catch (error) {
+        failed = true
+        throw error
+      } finally {
+        NntpService.release(conn, failed)
+      }
+
+      const meta = parseYencMeta(body)
+      const totalParts = inferTotalParts(meta)
+      Object.assign(update, {
+        yenc_name: meta.begin.name,
+        size: meta.begin.size,
+        total_parts: totalParts,
+        sampled_part: meta.begin.part ?? 1,
+      })
+      if ((meta.begin.part ?? 1) === 1) {
+        const data = decodeSegment(body, { lenientSize: true, verifyCrc: false }).data
+        update.content_type = sniffContentType(data)
+        const rar = parseRarHeaders(data)
+        if (rar) update.encrypted = rar.headersEncrypted || rar.entries.some((e) => e.encrypted)
+      }
+
+      update.status = 'sampled'
+      if (totalParts <= indexerConfig.deferredExpandMaxParts && update.encrypted !== true) {
+        const queued = await this.expand(file)
+        if (queued > 0) update.status = 'expanded'
+      }
+    } catch (error: any) {
+      update.status = 'failed'
+      update.error = String(error.message).slice(0, 500)
+    }
+    await db.from('deferred_files').where('id', file.id).update(update)
+    return update.status as string
+  }
+
+  /** Finds the file's articles again (XOVER over its range) and queues their bodies. */
+  private async expand(file: any): Promise<number> {
+    const first = Number(file.first_article)
+    const last = Number(file.last_article)
+    if (last - first > EXPAND_MAX_RANGE) return 0
+
+    const conn = await NntpService.acquire()
+    let failed = false
+    let overviews: [number, OverviewHeader][]
+    try {
+      await conn.group(file.group_name)
+      overviews = (await conn.xover(String(first), String(last))).overviews
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    const jobs = overviews
+      .filter(([, header]) => header && deferredName(header.subject) === file.name)
+      .map(([articleNumber, header]) => bodyJob(header, file.group_name, articleNumber))
+    if (jobs.length > 0) await QueueService.bodyQueue.addBulk(jobs)
+    return jobs.length
+  }
+}
+
+export default new DeferredService()

app/services/FetchRangeService.ts

@@ -0,0 +1,118 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
+
+/** A failed range is retried after this long... */
+const REQUEUE_AFTER_MS = 5 * 60 * 1000
+/** ...at most this many times; after that it stays visible as a gap. */
+const MAX_REQUEUES = 5
+/** A queued range without a job in Redis (e.g. Redis was flushed) is re-queued after this long. */
+const LOST_AFTER_MS = 60 * 60 * 1000
+
+interface Group {
+  id: number
+  name: string
+}
+
+/**
+ * Tracks the article ranges given to worker:fetch. A range only counts as
+ * indexed once the worker reports it done, so failed or lost fetch jobs are
+ * retried instead of silently skipped. See CONCEPT.md (D18).
+ */
+class FetchRangeService {
+  /** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
+  async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
+    const now = new Date()
+    const [row] = await db
+      .table('fetch_ranges')
+      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
+      .returning(['id', 'requeues'])
+    await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
+  }
+
+  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
+    await QueueService.nntpFetchQueue.add(
+      'fetch-headers',
+      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
+      // The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
+      { jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
+    )
+  }
+
+  private jobId(rangeId: number, requeues: number) {
+    return `range-${rangeId}-${requeues}`
+  }
+
+  async markDone(rangeId: number) {
+    await db.from('fetch_ranges').where('id', rangeId).update({ status: 'done', error: null, updated_at: new Date() })
+  }
+
+  async markFailed(rangeId: number, error: string) {
+    await db.from('fetch_ranges').where('id', rangeId).update({ status: 'failed', error, updated_at: new Date() })
+  }
+
+  /** Re-queues failed ranges, and queued ranges whose job no longer exists. */
+  async requeue(group: Group): Promise<number> {
+    const now = Date.now()
+    const candidates = await db
+      .from('fetch_ranges')
+      .where('group_id', group.id)
+      .where('requeues', '<', MAX_REQUEUES)
+      .where((q) =>
+        q
+          .where((f) => f.where('status', 'failed').where('updated_at', '<', new Date(now - REQUEUE_AFTER_MS)))
+          .orWhere((l) => l.where('status', 'queued').where('updated_at', '<', new Date(now - LOST_AFTER_MS)))
+      )
+      .orderBy('start_id')
+
+    let requeued = 0
+    for (const range of candidates) {
+      if (range.status === 'queued') {
+        const job = await QueueService.nntpFetchQueue.getJob(this.jobId(Number(range.id), range.requeues))
+        const state = job ? await job.getState() : 'missing'
+        if (state !== 'missing' && state !== 'failed' && state !== 'unknown') continue
+      }
+      await db
+        .from('fetch_ranges')
+        .where('id', range.id)
+        .update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
+      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
+      requeued++
+      logger.warn(
+        `Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
+      )
+    }
+    return requeued
+  }
+
+  /**
+   * Moves `groups.last_indexed_id` up to just below the first live range
+   * that is not done yet (backfill ranges lie below it and don't count), and
+   * forgets done ranges. Returns the new watermark.
+   */
+  async advanceWatermark(group: Group): Promise<bigint> {
+    const { rows } = await db.rawQuery(
+      `UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
+         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
+         g.last_queued_id))
+       WHERE g.id = ? RETURNING last_indexed_id`,
+      [group.id]
+    )
+    await db.from('fetch_ranges').where('group_id', group.id).where('status', 'done').delete()
+    return BigInt(rows[0].last_indexed_id)
+  }
+
+  /** Ranges that failed for good (for monitoring). */
+  async abandoned(group: Group) {
+    return db
+      .from('fetch_ranges')
+      .where('group_id', group.id)
+      .where('status', 'failed')
+      .where('requeues', '>=', MAX_REQUEUES)
+      .count('* as count')
+      .first()
+  }
+}
+
+export default new FetchRangeService()

app/services/FileStore.ts

@@ -0,0 +1,55 @@
+import db from '@adonisjs/lucid/services/db'
+import type { QueryClientContract } from '@adonisjs/lucid/types/database'
+import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
+
+/**
+ * The only code that knows how `files` keeps its segment lists (CONCEPT.md
+ * D31): compressed in `files.segments`, with the first segment's message-id
+ * in the indexed `first_message_id` for lookups. Read them with
+ * `segmentsOf(file)`; write new files with `FileStore.insert`.
+ */
+
+/** A new `files` row, with its segments as a list. */
+export type NewFile = Record<string, unknown> & { segments: Segment[] }
+
+const unpacked = new WeakMap<object, Segment[]>()
+
+/** The segments of a `files` row (a row from `db.from('files')`), in order. */
+export function segmentsOf(file: { segments?: Buffer | Segment[] | null }): Segment[] {
+  const { segments } = file
+  if (!segments) return []
+  if (Array.isArray(segments)) return segments
+  let list = unpacked.get(file)
+  if (!list) {
+    list = unpackSegments(segments)
+    unpacked.set(file, list)
+  }
+  return list
+}
+
+/** Stored form of a segment list, for the File model's column. */
+export function storedSegments(segments: Segment[]) {
+  return packSegments(segments)
+}
+
+export function loadedSegments(stored: Buffer | null): Segment[] {
+  return stored ? unpackSegments(stored) : []
+}
+
+class FileStore {
+  /**
+   * Inserts files, skipping those whose `file_key` exists. Returns the rows
+   * that were inserted (id and filename).
+   */
+  async insert(files: NewFile[], client: Pick<QueryClientContract, 'knexQuery'> = db): Promise<{ id: number; filename: string }[]> {
+    if (files.length === 0) return []
+    const rows = files.map(({ segments, ...file }) => {
+      const sorted = [...segments].sort((a, b) => a.number - b.number)
+      return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
+    })
+    const inserted = await client.knexQuery().table('files').insert(rows).onConflict('file_key').ignore().returning(['id', 'filename'])
+    return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
+  }
+}
+
+export default new FileStore()

app/services/IndexMaintenance.ts

@@ -0,0 +1,85 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Only indexes at least this big are worth checking. */
+const MIN_BYTES = 50 * 1024 * 1024
+/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
+const MAX_LEAF_DENSITY = 50
+
+export interface Rebuilt {
+  index: string
+  density: number
+  before: number
+  after: number
+}
+
+/**
+ * GC deletes millions of rows (single-article binaries, copied parts, empty
+ * releases). Vacuum makes their index entries reusable but doesn't compact
+ * the index, so B-trees on the churning tables end up 10–20% full (LOG.md
+ * 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
+ * D33). Needs the pgstattuple extension.
+ */
+class IndexMaintenance {
+  async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
+    const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
+    if (extension.length === 0) {
+      logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
+      return []
+    }
+    if (!dryRun) await this.dropLeftovers()
+
+    const { rows: candidates } = await db.rawQuery(
+      `SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
+       FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
+       JOIN pg_namespace n ON n.oid = c.relnamespace
+       WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
+       ORDER BY 2 DESC`,
+      [MIN_BYTES]
+    )
+    const rebuilt: Rebuilt[] = []
+    for (const { index, bytes } of candidates) {
+      const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
+      const density = Number(rows[0].avg_leaf_density)
+      if (density >= MAX_LEAF_DENSITY) continue
+      if (!dryRun && !(await this.reindex(index))) continue
+      const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
+      const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
+      rebuilt.push(result)
+      logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
+    }
+    return rebuilt
+  }
+
+  /**
+   * REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
+   * binaries during a backfill). Its half-built copy is dropped and it is
+   * tried once more; a second failure skips the index until the next run.
+   */
+  /** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
+  private async dropLeftovers(index = '') {
+    const { rows } = await db.rawQuery(
+      `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
+       WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
+      [index]
+    )
+    for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
+  }
+
+  private async reindex(index: string): Promise<boolean> {
+    for (let attempt = 1; attempt <= 2; attempt++) {
+      try {
+        await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
+        return true
+      } catch (error: any) {
+        logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
+        await this.dropLeftovers(index)
+      }
+    }
+    return false
+  }
+}
+
+const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
+
+export default new IndexMaintenance()

app/services/IngestService.ts

@@ -0,0 +1,471 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import type { QueryClientContract } from '@adonisjs/lucid/types/database'
+import type { Knex } from 'knex'
+import QueueService from '#services/QueueService'
+import NameService from '#services/NameService'
+import { parseXrefGroups } from '#lib/nntp/xref'
+import { fileKey, parseSubject, weakFileKey } from '#lib/subject/index'
+import { releaseIdentity, type ReleaseIdentity } from '#lib/release/index'
+import { inferTotalParts, type YencMeta } from '#lib/yenc/index'
+
+/** One segment, with the identity of the file it belongs to already resolved. */
+export interface IngestSegment {
+  fileKey: string
+  poster: string
+  /** Subject without segment counter. */
+  subject: string
+  name: string
+  totalParts: number
+  number: number
+  /** Without angle brackets. */
+  messageId: string
+  bytes: number
+  articleNumber?: bigint | number
+  postedAt?: Date
+  groups: string[]
+  /**
+   * The identity comes from the yEnc name only (obfuscated posts). Such names
+   * can be shared by several equal-sized files of one post, so a second
+   * article for the same part number marks the binary as conflicted.
+   */
+  weakIdentity?: boolean
+  /** Release (download) this file belongs to, when it can be determined. */
+  release?: ReleaseIdentity | null
+  /** Exact file size from =ybegin (body-derived segments only). */
+  size?: number
+  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
+  pcrc32?: string
+  partSize?: number
+  fileCrc32?: string
+  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
+  md5_16k?: string
+}
+
+export interface CompletedBinary {
+  id: number
+  fileKey: string
+}
+
+const BINARY_CHUNK = 1000
+/** last_seen_at is only rewritten when it is older than this (it drives GC at 48 h). */
+const TOUCH_MS = 10 * 60 * 1000
+
+type Merge = 'least' | 'coalesce' | 'touch'
+
+/**
+ * Inserts rows keyed by `key` and returns their ids. Existing rows are not
+ * locked unless something in them changes (`least`: an earlier value,
+ * `coalesce`: a value where there was none, `touch`: a timestamp more than
+ * TOUCH_MS newer). Parallel fetch jobs see mostly the same rows, and
+ * `INSERT … ON CONFLICT DO UPDATE` made them wait on each other for every one.
+ */
+async function upsertKeyed(
+  knex: Knex,
+  table: 'binaries' | 'releases',
+  key: string,
+  rows: Record<string, unknown>[],
+  merge: Record<string, [Merge, string]>
+): Promise<Map<string, number>> {
+  const ids = new Map<string, number>()
+  if (rows.length === 0) return ids
+  const inserted: Record<string, any>[] = await knex(table).insert(rows).onConflict(key).ignore().returning(['id', key])
+  for (const row of inserted) ids.set(row[key], Number(row.id))
+
+  const incoming = new Map(rows.filter((r) => !ids.has(r[key] as string)).map((r) => [r[key] as string, r]))
+  if (incoming.size === 0) return ids
+  const columns = Object.keys(merge)
+  const existing: Record<string, any>[] = await knex(table).whereIn(key, [...incoming.keys()]).select('id', key, ...columns)
+
+  const changes: Record<string, unknown>[] = []
+  for (const row of existing) {
+    ids.set(row[key], Number(row.id))
+    const next = incoming.get(row[key])!
+    const changed = columns.some((column) => {
+      const [kind] = merge[column]
+      const current = row[column]
+      const value = next[column]
+      if (value === null || value === undefined) return false
+      if (kind === 'coalesce') return current === null
+      if (kind === 'least') return current === null || value < current
+      return current === null || +(value as Date) - +current > TOUCH_MS
+    })
+    if (changed) changes.push({ id: Number(row.id), ...Object.fromEntries(columns.map((c) => [c, next[c] ?? null])) })
+  }
+  if (changes.length === 0) return ids
+
+  const set = columns
+    .map((c) => {
+      const [kind] = merge[c]
+      const fn = kind === 'least' ? 'least' : kind === 'touch' ? 'greatest' : 'coalesce'
+      return `${c} = ${fn}(t.${c}, v.${c})`
+    })
+    .join(', ')
+  const types = columns.map((c) => `${c} ${merge[c][1]}`).join(', ')
+  changes.sort((a, b) => (a.id as number) - (b.id as number))
+  for (let attempt = 1; ; attempt++) {
+    try {
+      await knex.raw(
+        `UPDATE ${table} t SET ${set}
+         FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, ${types})
+         WHERE t.id = v.id`,
+        [JSON.stringify(changes)]
+      )
+      return ids
+    } catch (error: any) {
+      // Rare now, but two jobs can still update overlapping rows in a different order.
+      if (error.code !== '40P01' || attempt >= 3) throw error
+    }
+  }
+}
+/** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
+export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
+/** A par2 set and its files are posted together; don't match across days. */
+export const PAR2_MATCH_WINDOW_SECONDS = 12 * 60 * 60
+const PART_CHUNK = 2000
+
+function chunks<T>(items: T[], size: number): T[][] {
+  const result: T[][] = []
+  for (let i = 0; i < items.length; i += size) result.push(items.slice(i, i + size))
+  return result
+}
+
+function validDate(date?: Date) {
+  return date && !Number.isNaN(date.getTime()) ? date : null
+}
+
+/** An XOVER overview entry as returned by nntp-js. */
+export type OverviewHeader = Record<string, string>
+
+function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
+  const xrefGroups = parseXrefGroups(header.xref)
+  return {
+    poster: header.from ?? '',
+    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
+    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
+    articleNumber,
+    postedAt: header.date ? new Date(header.date) : undefined,
+    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
+  }
+}
+
+/** Builds a segment from the subject line; null when the subject is obfuscated. */
+export function segmentFromHeader(
+  header: OverviewHeader,
+  groupName: string,
+  articleNumber?: bigint | number
+): IngestSegment | null {
+  const parsed = header.subject ? parseSubject(header.subject) : null
+  if (!parsed || !header['message-id']) return null
+  const base = common(header, groupName, articleNumber)
+  return {
+    ...base,
+    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
+    subject: parsed.base,
+    name: parsed.name,
+    totalParts: parsed.totalSegments,
+    number: parsed.segment,
+    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
+  }
+}
+
+/**
+ * Builds a segment for an obfuscated post from the yEnc header in its body.
+ * Subject and poster can be random per article, so the identity comes from
+ * the yEnc header only (see weakFileKey).
+ */
+export function segmentFromYenc(
+  header: OverviewHeader,
+  groupName: string,
+  meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
+  articleNumber?: bigint | number
+): IngestSegment | null {
+  const base = common(header, groupName, articleNumber)
+  const totalParts = inferTotalParts(meta)
+  const number = meta.begin.part ?? 1
+  if (!base.messageId || number < 1 || number > totalParts) return null
+  const subject = `"${meta.begin.name}" yEnc`
+  return {
+    ...base,
+    fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
+    subject,
+    name: meta.begin.name,
+    totalParts,
+    number,
+    weakIdentity: true,
+    release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
+    size: meta.begin.size,
+    // Single-part posts only carry crc32, which is then also the part's CRC.
+    pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
+    partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
+    fileCrc32: meta.end?.crc32,
+  }
+}
+
+/**
+ * Stores segments in `binaries`/`parts` and queues every binary that became
+ * complete exactly once. Idempotent: ingesting the same segments again is a
+ * no-op. See CONCEPT.md (D12).
+ */
+class IngestService {
+  async ingest(segments: IngestSegment[]): Promise<CompletedBinary[]> {
+    if (segments.length === 0) return []
+
+    // Short statements, no enclosing transaction: every step is idempotent, and
+    // row locks held for a whole batch made parallel fetch jobs wait on each other.
+    const binaryIds = await (async () => {
+      const knex = db.connection().getWriteClient()
+      const now = new Date()
+      const releaseIds = await this.upsertReleases(knex, segments, now)
+
+      // One row per file_key (ON CONFLICT cannot touch a row twice per statement),
+      // sorted so concurrent workers lock rows in the same order and cannot deadlock.
+      const binaries = new Map<string, Record<string, unknown>>()
+      for (const segment of segments) {
+        const postedAt = validDate(segment.postedAt)
+        const existing = binaries.get(segment.fileKey)
+        if (existing) {
+          if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
+            existing.posted_at = postedAt
+          }
+          continue
+        }
+        binaries.set(segment.fileKey, {
+          file_key: segment.fileKey,
+          poster: segment.poster,
+          subject: segment.subject,
+          name: segment.name,
+          total_parts: segment.totalParts,
+          groups: JSON.stringify(segment.groups),
+          posted_at: postedAt,
+          first_seen_at: now,
+          last_seen_at: now,
+          release_id: segment.release ? releaseIds.get(segment.release.key) : null,
+          size: segment.size ?? null,
+          md5_16k: segment.md5_16k ?? null,
+        })
+      }
+      const binaryRows = [...binaries.values()].sort((a, b) =>
+        (a.file_key as string).localeCompare(b.file_key as string)
+      )
+
+      const idsByKey = new Map<string, number>()
+      for (const chunk of chunks(binaryRows, BINARY_CHUNK)) {
+        const ids = await upsertKeyed(knex, 'binaries', 'file_key', chunk, {
+          last_seen_at: ['touch', 'timestamptz'],
+          posted_at: ['least', 'timestamptz'],
+          release_id: ['coalesce', 'bigint'],
+          md5_16k: ['coalesce', 'varchar'],
+        })
+        for (const [key, id] of ids) idsByKey.set(key, id)
+      }
+
+      const partRows = segments
+        .map((segment) => ({
+          binary_id: idsByKey.get(segment.fileKey)!,
+          number: segment.number,
+          message_id: segment.messageId,
+          bytes: segment.bytes,
+          article_number: segment.articleNumber?.toString() ?? null,
+          pcrc32: segment.pcrc32 ?? null,
+          part_size: segment.partSize ?? null,
+          file_crc32: segment.fileCrc32 ?? null,
+          md5_16k: segment.md5_16k ?? null,
+        }))
+      const weakRows = partRows.filter((_, i) => segments[i].weakIdentity)
+
+      partRows.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
+      for (const chunk of chunks(partRows, PART_CHUNK)) {
+        await knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore()
+      }
+
+      if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
+
+      return [...idsByKey.values()]
+    })()
+
+    // par2 naming needs a size, which only body-derived segments have; for
+    // header batches these queries were most of Postgres' work.
+    const sized = [...new Set(segments.filter((s) => s.size != null).map((s) => s.fileKey))]
+    if (sized.length > 0) {
+      const sizedIds = (await db.from('binaries').whereIn('file_key', sized).select('id')).map((r) => Number(r.id))
+      await this.matchPar2Releases(sizedIds)
+      await NameService.resolve(sizedIds)
+    }
+
+    // After commit, so the last writer of a binary always sees all of its parts.
+    const completed = await this.claimCompleted(binaryIds)
+    await this.enqueue(completed)
+    return completed
+  }
+
+  /**
+   * Creates or touches the releases referenced by `segments` and returns
+   * their ids by release key. Also used by `release:rebuild`.
+   */
+  async upsertReleases(
+    knex: Knex,
+    segments: Pick<IngestSegment, 'release' | 'poster' | 'groups' | 'postedAt' | 'weakIdentity'>[],
+    now = new Date()
+  ): Promise<Map<string, number>> {
+    const releases = new Map<string, Record<string, unknown>>()
+    for (const segment of segments) {
+      const release = segment.release
+      if (!release) continue
+      const postedAt = validDate(segment.postedAt)
+      const existing = releases.get(release.key)
+      if (existing) {
+        if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
+          existing.posted_at = postedAt
+        }
+        // Not every file carries the [i/M] counter (e.g. a par2 index): take it from any that does.
+        if (existing.file_total === null && release.fileTotal) existing.file_total = release.fileTotal
+        continue
+      }
+      releases.set(release.key, {
+        release_key: release.key,
+        name: release.name,
+        // Obfuscating posters randomise the poster, so it says nothing about the release.
+        poster: segment.weakIdentity ? null : segment.poster,
+        file_total: release.fileTotal ?? null,
+        source: release.source,
+        groups: JSON.stringify(segment.groups),
+        posted_at: postedAt,
+        first_seen_at: now,
+        last_seen_at: now,
+      })
+    }
+
+    const ids = new Map<string, number>()
+    const rows = [...releases.values()].sort((a, b) =>
+      (a.release_key as string).localeCompare(b.release_key as string)
+    )
+    for (const chunk of chunks(rows, BINARY_CHUNK)) {
+      const result = await upsertKeyed(knex, 'releases', 'release_key', chunk, {
+        last_seen_at: ['touch', 'timestamptz'],
+        posted_at: ['least', 'timestamptz'],
+        file_total: ['coalesce', 'integer'],
+        source: ['coalesce', 'varchar'],
+      })
+      for (const [key, id] of result) ids.set(key, id)
+    }
+    return ids
+  }
+
+  /**
+   * Assigns body-derived binaries without a release and without a
+   * fingerprint to a known par2 set by size: only when that size is listed
+   * once in the set, the file is large, and exactly one set nearby lists it.
+   * Binaries with a fingerprint are matched exactly by NameService instead.
+   */
+  async matchPar2Releases(binaryIds: number[]) {
+    if (binaryIds.length === 0) return
+    await db.rawQuery(
+      `UPDATE binaries b SET release_id = m.release_id
+       FROM (
+         SELECT b2.id, min(rf.release_id) AS release_id
+         FROM binaries b2
+         JOIN release_files rf ON rf.size = b2.size
+         JOIN releases r ON r.id = rf.release_id
+         WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
+           AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
+           AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
+         GROUP BY b2.id
+         HAVING count(DISTINCT rf.release_id) = 1
+       ) m
+       WHERE b.id = m.id`,
+      [binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
+    )
+  }
+
+  /**
+   * Flags binaries where a stored part has a different message-id and a
+   * different part CRC than the incoming one: two files were merged under
+   * one identity. Identical reposts (same CRC) are ignored.
+   */
+  private async flagConflicts(
+    trx: QueryClientContract,
+    rows: {
+      binary_id: number
+      number: number
+      message_id: string
+      bytes: number
+      article_number: string | null
+      pcrc32: string | null
+      part_size: number | null
+      file_crc32: string | null
+      md5_16k: string | null
+    }[]
+  ) {
+    const bindings = [
+      rows.map((r) => r.binary_id),
+      rows.map((r) => r.number),
+      rows.map((r) => r.message_id),
+      rows.map((r) => r.bytes),
+      rows.map((r) => r.article_number),
+      rows.map((r) => r.pcrc32),
+      rows.map((r) => r.part_size),
+      rows.map((r) => r.file_crc32),
+      rows.map((r) => r.md5_16k),
+    ]
+    // Keep the extra articles: SplitService separates the volumes from them.
+    const conflicts = await trx.rawQuery(
+      `INSERT INTO part_conflicts (binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
+       SELECT v.binary_id, v.number, v.message_id, v.bytes, v.article_number, v.pcrc32, v.part_size, v.file_crc32, v.md5_16k
+       FROM unnest(?::bigint[], ?::int[], ?::text[], ?::int[], ?::bigint[], ?::text[], ?::int[], ?::text[], ?::text[])
+         AS v(binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
+       JOIN parts p ON p.binary_id = v.binary_id AND p.number = v.number
+       WHERE p.message_id <> v.message_id
+         -- Same part CRC: the same data posted again (a repost), not another file.
+         AND (p.pcrc32 IS NULL OR v.pcrc32 IS NULL OR p.pcrc32 <> v.pcrc32)
+       ON CONFLICT DO NOTHING
+       RETURNING binary_id`,
+      bindings as any
+    )
+    const binaryIds: number[] = [
+      ...new Set<number>(conflicts.rows.map((r: { binary_id: string }) => Number(r.binary_id))),
+    ]
+    if (binaryIds.length === 0) return
+    const result = await trx.rawQuery(
+      `UPDATE binaries SET conflicted_at = now()
+       WHERE conflicted_at IS NULL AND id = ANY(?) RETURNING name`,
+      [binaryIds]
+    )
+    for (const row of result.rows) {
+      logger.warn(`Binary "${row.name}" has conflicting parts; it will be split into volumes.`)
+    }
+  }
+
+  /**
+   * Marks complete binaries among `binaryIds` as claimed. The single UPDATE
+   * locks each row, so concurrent callers never claim the same binary twice.
+   */
+  async claimCompleted(binaryIds: number[]): Promise<CompletedBinary[]> {
+    if (binaryIds.length === 0) return []
+    const result = await db.rawQuery(
+      `UPDATE binaries b SET completed_at = now()
+       WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
+         AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) >= b.total_parts
+       RETURNING b.id, b.file_key`,
+      [binaryIds]
+    )
+    return result.rows.map((row: { id: string; file_key: string }) => ({
+      id: Number(row.id),
+      fileKey: row.file_key,
+    }))
+  }
+
+  async enqueue(completed: CompletedBinary[]) {
+    if (completed.length === 0) return
+    // jobId = file_key: BullMQ ignores an add() for an id that already exists.
+    await QueueService.fileQueue.addBulk(
+      completed.map(({ id, fileKey }) => ({
+        name: 'process-file',
+        data: { binaryId: id },
+        opts: { jobId: fileKey },
+      }))
+    )
+  }
+}
+
+export default new IngestService()

app/services/NameService.ts

@@ -0,0 +1,128 @@
+import db from '@adonisjs/lucid/services/db'
+import { looksRandom } from '#services/ArchiveService'
+
+/**
+ * A release that is nothing more than one randomly named file (its own
+ * filename stem). Such a file moves into the par2 set that names it exactly;
+ * real groupings (par2, RAR, NZB, spot, subject, post id) are left alone.
+ */
+export const WEAK_RELEASE_SQL = (column: string) =>
+  `EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem')
+     AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))`
+
+/**
+ * Whether `candidate` (e.g. from a par2 listing) should replace `current`:
+ * the current name is random, or a single token without separators
+ * (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the
+ * candidate reads like a release name.
+ */
+export function isBetterName(current: string, candidate: string): boolean {
+  if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false
+  return looksRandom(current) || !/[._ -]/.test(current)
+}
+
+/** Smaller files (nfo, sfv) collide too easily on size alone. */
+const MIN_SIZE_ONLY = 64 * 1024
+
+/**
+ * Gives binaries (and their files) the real filename that a par2 set lists
+ * for them. See CONCEPT.md (D21).
+ * - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This
+ *   also puts the binary in that par2 set's release.
+ * - Size only, when the fingerprint is unknown: within the binary's own
+ *   release, only when exactly one listed file has that size.
+ */
+class NameService {
+  async resolve(binaryIds: number[]): Promise<number> {
+    if (binaryIds.length === 0) return 0
+    const exact = await db.rawQuery(
+      `UPDATE binaries b SET real_name = m.name,
+         release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END
+       FROM (
+         SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id
+         FROM binaries b2
+         JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k
+         WHERE b2.id = ANY(?)
+         ORDER BY b2.id, rf.release_id DESC
+       ) m
+       WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL
+         OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`,
+      [binaryIds]
+    )
+    const bySize = await db.rawQuery(
+      `UPDATE binaries b SET real_name = m.name
+       FROM (
+         SELECT b2.id, min(rf.name) AS name
+         FROM binaries b2
+         JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size
+         WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
+         GROUP BY b2.id
+         HAVING count(DISTINCT rf.name) = 1
+       ) m
+       WHERE b.id = m.id`,
+      [binaryIds, MIN_SIZE_ONLY]
+    )
+    await db.rawQuery(
+      `UPDATE files f SET real_name = b.real_name,
+         release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END
+       FROM binaries b
+       WHERE b.id = ANY(?) AND b.file_id = f.id
+         AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL)
+           OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`,
+      [binaryIds]
+    )
+    return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0)
+  }
+
+  /**
+   * When a par2 set arrives: the binaries it can name, i.e. those matching
+   * one of its entries exactly (size + first-16-KiB MD5) and those in its
+   * release. Selecting every binary of the listed sizes instead picked up
+   * hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)).
+   */
+  async resolveForRelease(releaseId: number): Promise<number> {
+    const { rows } = await db.rawQuery(
+      `SELECT b.id FROM release_files rf
+       JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size
+       WHERE rf.release_id = ?
+       UNION
+       SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`,
+      [releaseId, releaseId]
+    )
+    return this.resolve(rows.map((r: { id: string }) => Number(r.id)))
+  }
+}
+
+/**
+ * Repair for data from before files moved into the par2 set that names them
+ * (used by `release:rename`): regroups exact matches out of random-named
+ * single-file releases, then renames par2 releases that still carry a
+ * random name after their listing. No downloads.
+ */
+export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> {
+  const { rows } = await db.rawQuery(
+    `SELECT DISTINCT b.id FROM binaries b
+     JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size
+     WHERE b.release_id IS DISTINCT FROM rf.release_id
+       AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})`
+  )
+  const ids = rows.map((r: { id: string }) => Number(r.id))
+  for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000))
+
+  const { par2ReleaseName } = await import('#lib/release/index')
+  const listings = await db.rawQuery(
+    `SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
+     JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
+  )
+  let renamed = 0
+  for (const release of listings.rows) {
+    const better = par2ReleaseName(release.names)
+    if (!better || better === release.name || !isBetterName(release.name, better)) continue
+    await db.from('releases').where('id', release.id).update({ name: better })
+    renamed++
+  }
+  return { regrouped: ids.length, renamed }
+}
+
+const nameService = new NameService()
+export default nameService

app/services/NntpService.ts

@@ -0,0 +1,133 @@
+import nntpConfig from '#config/nntp'
+import { NNTP } from "#lib/nntp/nntp_js";
+import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
+
+class NntpService {
+  private readonly poolSize: number;
+  private allConnections: Set<any>;
+  private idleConnections: any[];
+  private waiters: ((conn: any) => void)[];
+  private createdCount: number;
+
+  constructor(poolSize = 10) {
+    this.poolSize = poolSize;
+    this.allConnections = new Set();
+    this.idleConnections = [];
+    this.waiters = [];
+    this.createdCount = 0;
+  }
+
+  private async _createConnection() {
+    const conn = new NNTP(nntpConfig.host, nntpConfig.port);
+    try {
+      await conn.connect();
+      if (nntpConfig.user) {
+        await conn.login(nntpConfig.user, nntpConfig.password?.release());
+      }
+    } catch (error) {
+      // Free the slot, otherwise a failed connect shrinks the pool for good.
+      this.createdCount--;
+      this._destroy(conn);
+      throw error;
+    }
+    this.allConnections.add(conn);
+    return conn;
+  }
+
+  public async acquire() {
+    if (this.idleConnections.length > 0) {
+      return this.idleConnections.pop();
+    }
+
+    if (this.createdCount < this.poolSize) {
+      this.createdCount++;
+      console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`);
+      return this._createConnection();
+    }
+
+    return new Promise(resolve => this.waiters.push(resolve));
+  }
+
+  /**
+   * Returns a connection to the pool. Pass `failed` when a command on it
+   * threw: the connection may be dead (e.g. closed by the server while idle),
+   * so it is discarded instead of being handed to the next job.
+   */
+  public release(conn: any, failed = false) {
+    if (failed) {
+      this.discard(conn);
+      return;
+    }
+    const resolve = this.waiters.shift();
+    if (resolve) {
+      resolve(conn);
+    } else {
+      this.idleConnections.push(conn);
+    }
+  }
+
+  /** Closes a broken connection and frees its slot for a new one. */
+  public discard(conn: any) {
+    this.allConnections.delete(conn);
+    this.createdCount--;
+    this._destroy(conn);
+
+    // Someone is waiting for a slot: open a fresh connection for them.
+    const resolve = this.waiters.shift();
+    if (resolve) {
+      this.createdCount++;
+      this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
+    }
+  }
+
+  private _destroy(conn: any) {
+    try {
+      conn.sock?.destroy();
+    } catch {
+      // already closed
+    }
+  }
+
+  /** Selects a group and returns its correctly parsed article range. */
+  public async selectGroup(conn: any, name: string): Promise<GroupInfo> {
+    const { resp } = await conn.group(name)
+    return parseGroupResponse(resp)
+  }
+
+  /**
+   * Rejects when `promise` hasn't settled within `ms`. nntp-js has no
+   * timeouts: a connection the server stops answering (still ESTABLISHED,
+   * nothing in flight) leaves its command waiting forever. The caller must
+   * then release the connection as failed, so it's destroyed.
+   */
+  public timed<T>(promise: Promise<T>, what: string, ms = nntpConfig.commandTimeoutMs): Promise<T> {
+    let timer: NodeJS.Timeout
+    const timeout = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`NNTP ${what}: no answer within ${ms / 1000} s`)), ms)
+    })
+    return Promise.race([promise, timeout]).finally(() => clearTimeout(timer))
+  }
+
+  /** Fetches a raw (still dot-stuffed) article body as a Buffer. */
+  public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
+    const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
+    const { data } = await conn.body(id)
+    return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
+  }
+
+  public async shutdown() {
+    console.log('Shutting down all connections in the pool.');
+    // A connection the server already closed may never answer QUIT.
+    const quit = (conn: any) =>
+      Promise.race([conn.quit(), new Promise((resolve) => setTimeout(resolve, 2000).unref())])
+        .catch(() => {})
+        .finally(() => this._destroy(conn));
+    await Promise.all([...this.allConnections].map(quit));
+    this.allConnections.clear();
+    this.idleConnections.length = 0;
+    this.waiters.length = 0;
+    this.createdCount = 0;
+  }
+}
+
+export default new NntpService(nntpConfig.poolSize);

app/services/NzbImportService.ts

@@ -0,0 +1,231 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import IngestService from '#services/IngestService'
+import { type ReleaseIdentity } from '#lib/release/index'
+import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
+import { parseSubject } from '#lib/subject/index'
+import { fileStem } from '#lib/release/index'
+import { looksRandom } from '#services/ArchiveService'
+import QueueService from '#services/QueueService'
+import FileStore from '#services/FileStore'
+import { isArchiveStart } from '#lib/content/index'
+
+export interface NzbImportResult {
+  releaseId: number
+  releaseName: string
+  listed: number
+  known: number
+  added: number
+  password: boolean
+}
+
+/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
+export function nzbFileName(subject: string): string {
+  const parsed = parseSubject(subject)
+  const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
+  if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
+  return unquotedFileName(subject)
+}
+
+/**
+ * A filename from a subject without quotes, as some spot NZBs have:
+ * `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
+ * `06 - Song.flac  Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
+ * trailing size are dropped; then the name ends at the first extension
+ * followed by a space.
+ */
+export function unquotedFileName(subject: string): string {
+  const cleaned = subject
+    .replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
+    .replace(/\s\d+\s*$/, ' ')
+    .replace(/(\s+-)+\s*$/, '')
+    .replace(/^\s*-\s+/, '')
+    .replace(/\s+/g, ' ')
+    .trim()
+  const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
+  const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
+  return name.replace(/\s+-$/, '').trim() || subject.trim()
+}
+
+export interface NzbImportOptions {
+  /**
+   * For NZBs inside an archive: the entry's name. It keeps several NZBs of
+   * one archive apart and names their release when the NZB has no name.
+   */
+  entryName?: string
+  /** Move the posted file into the release the NZB describes (default true). */
+  moveContainer?: boolean
+  /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
+  name?: string
+  /** Release source for new releases (default `nzb`). */
+  source?: ReleaseIdentity['source']
+  /**
+   * The NZB is authoritative for which files form the release and what it is
+   * called (Spotnet spots): its files get the NZB's own release, named
+   * `name`, also when name rules had put them in another release. Files
+   * another trusted NZB claimed first stay where they are.
+   */
+  trusted?: boolean
+}
+
+/**
+ * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
+ * Files it lists that are already known (by first segment) keep their
+ * release, which the NZB then joins; the others are added as files, so posts
+ * spread over groups we do not index still become complete.
+ */
+class NzbImportService {
+  async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
+    const nzb = parseNzb(xml)
+    if (!nzb || nzb.files.length === 0) return null
+
+    const entries = nzb.files.map((f) => ({
+      file: f,
+      name: nzbFileName(f.subject),
+      firstId: f.segments[0].id,
+    }))
+
+    const added: { id: number; filename: string }[] = []
+    const result = await db.transaction(async (trx) => {
+      const known = await trx
+        .from('files')
+        .whereIn('first_message_id', entries.map((e) => e.firstId))
+        .select('id', 'release_id', 'first_message_id')
+      const knownIds = new Set(known.map((f) => f.first_message_id))
+
+      // Join the release the listed files are already in, else the posted NZB's own, else a new one.
+      const counts = new Map<number, number>()
+      for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
+      let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
+      const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
+      const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
+      const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
+
+      // Releases of other trusted NZBs: their files are not taken over.
+      let claimed = new Set<number>()
+      if (options.trusted) {
+        // Only while that NZB still stands: a disposed spot's files go to its correction.
+        const holders = await trx
+          .from('releases as r')
+          .whereIn('r.id', [...counts.keys()])
+          .where('r.source', 'spot')
+          .whereNot('r.release_key', key)
+          .whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
+          .select('r.id')
+        claimed = new Set(holders.map((r) => Number(r.id)))
+        // Join another spot's release when it holds most of these files (a repost); else use our own.
+        if (releaseId && !claimed.has(releaseId)) releaseId = undefined
+        if (releaseId) claimed.delete(releaseId)
+      }
+      if (!releaseId) {
+        const ids = await IngestService.upsertReleases(trx.knexClient, [
+          {
+            release: { key, name: title, source: options.source ?? 'nzb' },
+            poster: nzbFile.poster,
+            groups: nzbFile.groups ?? [],
+            postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
+          },
+        ])
+        releaseId = ids.get(key)!
+      }
+
+      const now = new Date()
+      const missing = entries.filter((e) => !knownIds.has(e.firstId))
+      for (let i = 0; i < missing.length; i += 500) {
+        added.push(
+          ...(await FileStore.insert(
+            missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
+              file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
+              filename: name,
+              poster: file.poster || nzbFile.poster,
+              date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
+              parts: file.segments.length,
+              size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
+              segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
+              groups: JSON.stringify(file.groups),
+              release_id: releaseId,
+              created_at: now,
+              updated_at: now,
+            })),
+            trx
+          ))
+        )
+      }
+      // The poster's NZB says which files form the post: known files join this
+      // release, also when name rules had split them over several.
+      const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
+      const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
+      await trx
+        .from('files')
+        .whereIn('id', moving.map((f) => f.id))
+        .update({ release_id: releaseId })
+      if (previous.length) {
+        await trx.rawQuery(
+          `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
+          [releaseId, releaseId, releaseId]
+        )
+        await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
+      }
+      // The NZB file joins the release it describes, unless it is part of a real release already.
+      const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
+      if (own !== releaseId && options.moveContainer !== false) {
+        const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
+        if (!ownRelease || ownRelease.file_count <= 1) {
+          await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
+          if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
+        }
+      }
+
+      const release = await trx.from('releases').where('id', releaseId).first()
+      const update: Record<string, unknown> = {}
+      if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
+      if (!release.file_total) update.file_total = entries.length
+      if (options.trusted) {
+        // Only its own release: a repost joining another spot's release keeps that one's title.
+        if (release.release_key === key && release.name !== title) update.name = title
+        if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
+      } else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
+        update.name = title
+      }
+      if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
+
+      const result = {
+        releaseId,
+        releaseName: (update.name as string) ?? release.name,
+        listed: entries.length,
+        known: known.length,
+        added: missing.length,
+        password: Boolean(nzb.meta.password),
+      }
+      logger.info(
+        `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
+          `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
+      )
+      return result
+    })
+    // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
+    const archives = added.filter((f) => isArchiveStart(f.filename))
+    if (archives.length) {
+      await QueueService.collectionQueue.addBulk(
+        archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
+      )
+    }
+    return result
+  }
+
+  /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
+  private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
+    if (nzb.meta.name) return nzb.meta.name
+    const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
+    if (entry && !looksRandom(entry)) return entry
+    const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
+    if (own && !looksRandom(own)) return own
+    const largest = nzb.files
+      .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
+      .sort((a, b) => b.bytes - a.bytes)[0]
+    return fileStem(largest?.name ?? own)
+  }
+}
+
+export default new NzbImportService()

app/services/Par2Service.ts

@@ -0,0 +1,166 @@
+import db from '@adonisjs/lucid/services/db'
+import type { Segment } from '#lib/nzb/index'
+import logger from '@adonisjs/core/services/logger'
+import NntpService from '#services/NntpService'
+import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
+import { decodeSegment, YencFile } from '#lib/yenc/index'
+import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
+import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
+import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
+
+/** Par2 index files are small; recovery volumes can be large and are not needed. */
+const MAX_PARTS = 20
+/** Size-only matching below this is too ambiguous. */
+const SIZE_ONLY_MIN = 1024 * 1024
+
+function countBy<T>(values: T[]): Map<T, number> {
+  const counts = new Map<T, number>()
+  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
+  return counts
+}
+
+export interface Par2Result {
+  releaseId: number
+  releaseName: string
+  listed: number
+  matched: number
+}
+
+/**
+ * Recognises releases of obfuscated posts from their par2 files: the par2
+ * lists the real name and exact size of every file in the set, and
+ * body-derived binaries carry their exact size. See CONCEPT.md (D16).
+ */
+class Par2Service {
+  /**
+   * Reads a par2 file whose first segment is already decoded, downloading the
+   * remaining parts if there are any. Returns null when it is not par2.
+   */
+  async read(first: Buffer, segments: Segment[]): Promise<Par2Set | null> {
+    if (!isPar2(first)) return null
+    const ids = segments.map((part) => part.id)
+    if (ids.length > MAX_PARTS) return null
+    if (ids.length === 1) return parsePar2(first)
+
+    let assembled: YencFile | undefined
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      for (const id of ids) {
+        const segment = decodeSegment(await NntpService.fetchBody(conn, id))
+        assembled ??= YencFile.fromSegment(segment)
+        assembled.add(segment)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    return parsePar2(assembled!.getBuffer())
+  }
+
+  async attach(binary: any, set: Par2Set): Promise<Par2Result> {
+    const result = await db.transaction(async (trx) => {
+      // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
+      // the set into a release the files already belong to; size alone is
+      // ambiguous across posts (standard volume sizes recur).
+      const exact = await trx
+        .from('binaries')
+        .whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
+        .select('id', 'release_id')
+
+      // Size only, for files without a fingerprint: the size must be listed
+      // once in this set, be large, and belong to exactly one binary nearby.
+      const listedOnce = [...countBy(set.files.map((f) => f.length))]
+        .filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
+        .map(([size]) => size)
+      const bySize = listedOnce.length
+        ? await trx
+            .from('binaries')
+            .whereIn('size', listedOnce)
+            .whereNull('md5_16k')
+            .whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
+              binary.posted_at,
+              PAR2_MATCH_WINDOW_SECONDS,
+            ])
+            .select('id', 'release_id', 'size')
+        : []
+      const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
+      const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
+      const candidates = [...exact, ...fuzzy]
+
+      // Prefer a release the exactly matched files are already in (e.g.
+      // obfuscated volumes grouped by their stem); otherwise the set's own.
+      const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
+      let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
+      const identity = par2ReleaseIdentity(
+        set.recoverySetId,
+        set.files.map((f) => f.name)
+      )
+      if (!releaseId) {
+        const ids = await IngestService.upsertReleases(trx.knexClient, [
+          {
+            release: identity,
+            poster: binary.poster,
+            weakIdentity: true,
+            groups: binary.groups,
+            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
+          },
+        ])
+        releaseId = ids.get(identity.key)!
+      }
+
+      await trx
+        .knexClient('release_files')
+        .insert(
+          set.files.map((f) => ({
+            release_id: releaseId,
+            par2_file_id: f.fileId,
+            name: f.name,
+            size: f.length,
+            md5_16k: f.md5_16k,
+            md5: f.md5,
+            set_id: set.recoverySetId,
+          }))
+        )
+        .onConflict(['release_id', 'par2_file_id'])
+        .ignore()
+
+      const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
+      await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
+      // Exact matches sitting alone in a random-named release (one file of a
+      // season, each posted under its own random name) join the set's release.
+      await trx.rawQuery(
+        `UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
+        [releaseId, exact.map((c) => Number(c.id)), releaseId]
+      )
+      await trx.rawQuery(
+        `UPDATE files f SET release_id = b.release_id FROM binaries b
+         WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
+        [releaseId]
+      )
+
+      // The set may have joined a release named after one random filename.
+      const release = await trx.from('releases').where('id', releaseId).first()
+      let releaseName = release?.name
+      const better = par2ReleaseName(set.files.map((f) => f.name))
+      // A spot's title is kept (CONCEPT.md D29).
+      if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) {
+        await trx.from('releases').where('id', releaseId).update({ name: better })
+        releaseName = better
+      }
+      logger.info(
+        `Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
+          `${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
+      )
+      return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
+    })
+
+    // Real names for every known file of the set (exact fingerprint, or size within the release).
+    await NameService.resolveForRelease(result.releaseId)
+    return result
+  }
+}
+
+export default new Par2Service()

app/services/QueueService.ts

@@ -0,0 +1,61 @@
+import { Queue, type DefaultJobOptions } from 'bullmq'
+import queueConfig from '#config/queue'
+import gcConfig from '#config/gc'
+
+const defaultJobOptions: DefaultJobOptions = {
+  // Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 2000 },
+  removeOnComplete: {
+    age: gcConfig.jobs.completedAgeSeconds,
+    count: gcConfig.jobs.completedMaxCount,
+  },
+  removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
+}
+
+/**
+ * Queues are created (and connect to Redis) on first use. Ace imports every
+ * command at boot, so eager queues would keep one-off commands from exiting.
+ */
+class QueueService {
+  private readonly queues = new Map<string, Queue>()
+
+  private queue(name: string): Queue {
+    let queue = this.queues.get(name)
+    if (!queue) {
+      queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
+      this.queues.set(name, queue)
+    }
+    return queue
+  }
+
+  get nntpFetchQueue() {
+    return this.queue('nntp-fetch-queue')
+  }
+  get fileQueue() {
+    return this.queue('file-queue')
+  }
+  get bodyQueue() {
+    return this.queue('body-queue')
+  }
+  get collectionQueue() {
+    return this.queue('collection-queue')
+  }
+
+  /** Queues no longer used by the pipeline, kept here so GC can empty them. */
+  get legacy(): Queue[] {
+    return [this.queue('header-queue')]
+  }
+
+  get all(): Queue[] {
+    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue]
+  }
+
+  /** Closes the queues that were opened. */
+  async closeAll() {
+    await Promise.all([...this.queues.values()].map((queue) => queue.close()))
+    this.queues.clear()
+  }
+}
+
+export default new QueueService()

app/services/RedisService.ts

@@ -0,0 +1,25 @@
+import { createRequire } from 'module'
+import redisConfig from '#config/redis'
+
+const require = createRequire(import.meta.url)
+const IORedis = require('ioredis')
+
+/**
+ * Connects on first use. Ace imports every command at boot, so an eager
+ * connection would keep even `node ace list` from exiting.
+ */
+class RedisService {
+  private instance?: any
+
+  public get client(): any {
+    this.instance ??= new IORedis(redisConfig.connections.main)
+    return this.instance
+  }
+
+  async close() {
+    if (this.instance) await this.instance.quit()
+    this.instance = undefined
+  }
+}
+
+export default new RedisService()

app/services/ReleaseMergeService.ts

@@ -0,0 +1,42 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/**
+ * Uploads split per poster (CONCEPT.md D35). Some uploaders post every
+ * volume of a RAR set under a different poster; the post-id grouping then
+ * makes one release per volume, all with the same name
+ * (`Masters.of.the.Universe.1987…`: 191 one-file releases of one 199-volume
+ * set). Such releases are one upload when all their files share one stem and
+ * no file name repeats (a repost would repeat them, another upload would
+ * bring another stem); their files move into the oldest of them, and GC
+ * deletes the emptied ones.
+ */
+class ReleaseMergeService {
+  async mergeSplitPosts({ dryRun = false } = {}): Promise<{ names: number; releases: number }> {
+    const { rows } = await db.rawQuery(`
+      WITH dup AS (
+        SELECT name FROM releases
+        WHERE file_count > 0 AND source = 'post' AND is_readable_name(name)
+        GROUP BY name HAVING count(*) > 1
+      )
+      SELECT r.name, array_agg(DISTINCT r.id ORDER BY r.id) AS ids
+      FROM dup JOIN releases r ON r.name = dup.name AND r.file_count > 0 AND r.source = 'post'
+      JOIN files f ON f.release_id = r.id
+      GROUP BY r.name
+      HAVING count(DISTINCT name_stem(coalesce(f.real_name, f.filename))) = 1
+        AND count(f.id) = count(DISTINCT coalesce(f.real_name, f.filename))
+        AND max(r.posted_at) - min(r.posted_at) < interval '30 days'`)
+    let releases = 0
+    for (const { name, ids } of rows as { name: string; ids: (string | number)[] }[]) {
+      const [target, ...others] = ids.map(Number)
+      releases += others.length
+      if (dryRun) continue
+      await db.from('files').whereIn('release_id', others).update({ release_id: target })
+      await db.from('binaries').whereIn('release_id', others).update({ release_id: target })
+      logger.info(`Merged ${others.length + 1} releases of one upload split per poster into ${target} "${name}".`)
+    }
+    return { names: rows.length, releases }
+  }
+}
+
+export default new ReleaseMergeService()

app/services/SearchService.ts

@@ -0,0 +1,401 @@
+import db from '@adonisjs/lucid/services/db'
+import { buildNzb } from '#lib/nzb/index'
+import SpotService from '#services/SpotService'
+import { segmentsOf } from '#services/FileStore'
+import { spotCategoryLabels } from '#lib/spotnet/index'
+
+/**
+ * `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
+ * without asking); `missing`: the archive is encrypted and we don't have
+ * it; `none`: no password needed as far as we know.
+ */
+export type PasswordState = 'known' | 'missing' | 'none'
+
+export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
+  if (hasPassword) return 'known'
+  return encrypted ? 'missing' : 'none'
+}
+
+export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
+
+export interface SearchParams {
+  q?: string
+  /** Only releases whose name is readable (not an obfuscated token). */
+  readable?: boolean
+  /** Only releases that have all their files. */
+  complete?: boolean
+  /** Hide releases that need a password we don't have. */
+  hideEncrypted?: boolean
+  source?: string
+  poster?: string
+  sort?: SortOrder
+  page?: number
+  perPage?: number
+  /** Newznab: exact category ids (already expanded from parents). */
+  categories?: number[]
+  /** Newznab: leave out releases known to be missing files. */
+  notIncomplete?: boolean
+  maxAgeDays?: number
+  /** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
+  season?: number
+  episode?: number
+  /** Overrides `page` (Newznab uses offsets). */
+  offset?: number
+  /** Leave out releases that are only an encrypted container (nzbe). */
+  hideContainers?: boolean
+  /** Include incomplete releases that consist of par2 files only (hidden by default). */
+  includePar2Only?: boolean
+  /** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
+  countLimit?: number
+}
+
+const SORTS: Record<SortOrder, string> = {
+  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
+  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
+  largest: 'r.size DESC, r.id DESC',
+  files: 'r.file_count DESC, r.id DESC',
+  name: 'r.name ASC, r.id ASC',
+}
+
+/**
+ * Shown in search: releases with files, and Spotnet listings whose NZB is
+ * fetched on first download (CONCEPT.md D29). Matches the partial indexes.
+ */
+const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
+
+/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
+const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
+  SELECT 1 FROM files f WHERE f.release_id = r.id
+    AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
+
+/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
+function pattern(word: string): string {
+  const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
+  return `%${escaped.replace(/[._\- ]+/g, '_')}%`
+}
+
+/**
+ * As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
+ * spaces as one space. `The.Block.AU` and `the block au` search the same.
+ */
+export function searchNormalize(text: string): string {
+  return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
+}
+
+/** How long the web UI's totals are reused. */
+const STATS_TTL_MS = 60 * 1000
+
+/**
+ * Counting stops here: a broad search needn't count every match. Each
+ * counted row is a random heap read, so the web UI counts less (it only
+ * shows "1,000+"); Newznab clients page by this total and keep the default.
+ */
+const COUNT_LIMIT = 10_000
+
+/**
+ * Searching and browsing releases for the web interface (CONCEPT.md D22).
+ */
+class SearchService {
+  async search(params: SearchParams) {
+    const where: string[] = [
+      VISIBLE_SQL,
+      // Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
+      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
+    ]
+    const bindings: unknown[] = []
+
+    // Every word must occur in the release's search text: its name, archive
+    // listing and file names (CONCEPT.md D33), one trigram index for all.
+    const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
+    for (const word of words) {
+      where.push(`r.search_text LIKE ?`)
+      bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
+    }
+    if (params.readable) where.push('is_readable_name(r.name)')
+    if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
+    if (params.hideContainers) where.push('r.container IS NULL')
+    // par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
+    if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
+    // Kept current by triggers (migration …860).
+    if (params.complete) where.push('r.complete')
+    if (params.source) {
+      where.push('r.source = ?')
+      bindings.push(params.source)
+    }
+    if (params.poster) {
+      where.push('r.poster ILIKE ?')
+      bindings.push(pattern(params.poster))
+    }
+    if (params.categories?.length) {
+      where.push('r.category = ANY(?)')
+      bindings.push(params.categories)
+    }
+    if (params.notIncomplete) {
+      where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
+    }
+    if (params.maxAgeDays) {
+      where.push(`r.posted_at >= now() - make_interval(days => ?)`)
+      bindings.push(params.maxAgeDays)
+    }
+    if (params.season !== undefined) {
+      const s = params.season
+      const e = params.episode
+      where.push('r.name ~* ?')
+      bindings.push(
+        e !== undefined
+          ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
+          : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
+      )
+    }
+
+    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
+    const page = Math.max(params.page ?? 1, 1)
+    const offset = params.offset ?? (page - 1) * perPage
+    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
+    const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
+
+    const [{ rows }, { rows: countRows }] = await Promise.all([
+      db.rawQuery(
+        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
+                r.size, r.encrypted, is_readable_name(r.name) AS readable,
+                (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
+                (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
+                   SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
+                   WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
+                jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
+                r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
+                (SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
+         FROM releases r WHERE ${whereSql}
+         ORDER BY ${order} LIMIT ? OFFSET ?`,
+        [...bindings, perPage, offset]
+      ),
+      db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
+    ])
+
+    return {
+      page,
+      perPage,
+      offset,
+      total: Math.min(Number(countRows[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(countRows[0].total) > countLimit,
+      releases: rows.map((r: any) => ({
+        id: Number(r.id),
+        name: r.name,
+        source: r.source,
+        poster: r.poster,
+        groups: r.groups,
+        postedAt: r.posted_at,
+        fileTotal: r.file_total,
+        fileCount: r.file_count,
+        size: Number(r.size),
+        encrypted: r.encrypted,
+        readable: r.readable,
+        listedFiles: Number(r.listed_files),
+        listedPresent: Number(r.listed_present),
+        archiveEntries: Number(r.archive_entries),
+        complete: r.complete,
+        category: r.category,
+        container: r.container,
+        par2Only: r.par2_only,
+        listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
+        hasPassword: r.has_password,
+        passwordState: passwordState(r.encrypted, r.has_password),
+        spotDescription: r.spot_description,
+      })),
+    }
+  }
+
+  async release(id: number) {
+    const release = await db.from('releases').where('id', id).first()
+    if (!release) return null
+
+    const [files, listed, pending] = await Promise.all([
+      db
+        .from('files as f')
+        .leftJoin('binaries as b', 'b.file_id', 'f.id')
+        .where('f.release_id', id)
+        .orderByRaw('coalesce(f.real_name, f.filename)')
+        .select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
+      db.rawQuery(
+        `SELECT rf.name, rf.size,
+                EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
+                        WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
+         FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
+        [id]
+      ),
+      db.rawQuery(
+        `SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
+                coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
+         FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
+        [id]
+      ),
+    ])
+
+    const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
+    const spot = spotRow ? await SpotService.withDetails(spotRow) : null
+    return {
+      id: Number(release.id),
+      name: release.name,
+      source: release.source,
+      spot: spot
+        ? {
+            messageId: spot.message_id,
+            title: spot.title,
+            tag: spot.tag,
+            poster: spot.poster,
+            postedAt: spot.posted_at,
+            website: spot.website,
+            description: spot.description,
+            image: spot.image?.segments
+              ? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
+              : spot.image?.url
+                ? { external: spot.image.url }
+                : null,
+            categories: spotCategoryLabels(spot.category, spot.subcats),
+          }
+        : null,
+      password: release.password ?? null,
+      passwordSource: release.password_source ?? null,
+      passwordState: passwordState(release.encrypted, release.password !== null),
+      listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
+      poster: release.poster,
+      groups: release.groups,
+      postedAt: release.posted_at,
+      fileTotal: release.file_total,
+      fileCount: release.file_count,
+      size: Number(release.size),
+      encrypted: release.encrypted,
+      container: release.container,
+      contents: release.contents ?? [],
+      files: files.map((f) => ({
+        id: Number(f.id),
+        name: f.real_name ?? f.filename,
+        postedName: f.filename,
+        realName: f.real_name,
+        size: Number(f.exact_size ?? f.size),
+        parts: f.parts,
+        postedAt: f.date ? new Date(Number(f.date)) : null,
+        poster: f.poster,
+      })),
+      par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
+      pending: {
+        binaries: Number(pending.rows[0].binaries),
+        partsHave: Number(pending.rows[0].parts_have),
+        partsTotal: Number(pending.rows[0].parts_total),
+      },
+    }
+  }
+
+  /** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
+  async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
+    const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
+    if (!spot) return null
+    const data = await SpotService.image(spot)
+    if (!data) return null
+    const type =
+      data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
+      : data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
+      : data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
+      : data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
+      : data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
+      : null
+    return type ? { data, type } : null
+  }
+
+  async nzb(id: number): Promise<{ name: string; xml: string } | null> {
+    let release = await db.from('releases').where('id', id).first()
+    if (!release) return null
+    if (release.source === 'spot' && Number(release.file_count) === 0) {
+      // A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
+      const imported = await SpotService.importListing(id)
+      if (!imported) return null
+      id = imported
+      release = await db.from('releases').where('id', id).first()
+    }
+    const files = await db
+      .from('files')
+      .where('release_id', id)
+      .orderByRaw('coalesce(real_name, filename)')
+      .select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
+    const xml = buildNzb(
+      files.map((f) => ({
+        name: f.real_name ?? f.filename,
+        poster: f.poster,
+        date: Number(f.date),
+        groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
+        segments: segmentsOf(f),
+      })),
+      { title: release.name, password: release.password ?? undefined }
+    )
+    return { name: release.name, xml }
+  }
+
+  /**
+   * Totals for the web UI header. They count millions of rows (minutes with
+   * the backfill running), so they are computed at most once per
+   * STATS_TTL_MS and never twice at a time; callers in between share it.
+   */
+  async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= this.computeStats()
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        // A failed background refresh keeps the previous totals; only a caller that waits sees the error.
+        .catch((error) => {
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      // Only the very first caller waits; later ones get the previous totals meanwhile.
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
+   * from Postgres's statistics (visible releases = the entries of an index
+   * that covers exactly them), ratios and sizes from a 0.5% sample of the
+   * releases table. Within a few percent once autovacuum has analysed.
+   */
+  private async computeStats() {
+    const { rows } = await db.rawQuery(`
+      WITH sample AS (
+        SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
+        FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
+      SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
+             (SELECT reltuples FROM pg_class WHERE relname = 'files') AS files,
+             (SELECT count(*) FROM sample) AS sampled,
+             (SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
+             -- Indexed data only: a spot listing's size is what the spot announces.
+             (SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
+             (SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
+             (SELECT max(updated_at) FROM groups) AS last_indexed_at,
+             (SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
+                FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
+    const r = rows[0]
+    const releases = Math.max(0, Number(r.releases))
+    const sampled = Number(r.sampled)
+    // Sample counts scaled to all visible releases.
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    return {
+      approximate: true,
+      releases,
+      readableReleases: scale(r.sampled_readable),
+      files: Math.max(0, Number(r.files)),
+      size: scale(r.sampled_size),
+      encryptedReleases: scale(r.sampled_encrypted),
+      lastIndexedAt: r.last_indexed_at,
+      sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
+    }
+  }
+}
+
+export default new SearchService()

app/services/SevenZipService.ts

@@ -0,0 +1,202 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, open, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import NntpService from '#services/NntpService'
+import { segmentsOf } from '#services/FileStore'
+import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
+import type { RarInfo } from '#lib/rar/index'
+import {
+  checkHeader,
+  listingStart,
+  parseBsdtarListing,
+  parseSevenZipStart,
+  sevenZipVolume,
+  type SevenZipStart,
+} from '#lib/sevenzip/index'
+
+const run = promisify(execFile)
+
+/** The tail read for a listing: header plus packed header; more is not a header. */
+const MAX_TAIL_BYTES = 16 * 1024 * 1024
+
+/**
+ * Lists 7z archives, also split ones (.7z.001, .002, …), from the first
+ * segment of the first volume and the last segment(s) of the last one
+ * (CONCEPT.md D19). Those bytes are put at their place in a sparse file the
+ * size of the whole archive, which bsdtar (libarchive) lists: it only reads
+ * the start and the header at the end.
+ */
+class SevenZipService {
+  /**
+   * The volumes of the archive `file` belongs to, in order, when all of them
+   * up to the highest number seen are there; null otherwise. An unsplit (or
+   * unnamed) archive is its own only volume.
+   */
+  async volumes(file: any): Promise<any[] | null> {
+    const name = String(file.real_name ?? file.filename)
+    const volume = sevenZipVolume(name)
+    if (!volume) return [file]
+    if (!file.release_id) return null
+    const base = name.replace(/\.\d+$/, '').toLowerCase()
+    const rows = await db
+      .from('files')
+      .where('release_id', file.release_id)
+      .whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
+    const byNumber = new Map<number, any>()
+    for (const row of rows) {
+      const n = sevenZipVolume(String(row.real_name ?? row.filename))
+      if (n && !byNumber.has(n)) byNumber.set(n, row)
+    }
+    const max = Math.max(0, ...byNumber.keys())
+    const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
+    return max > 0 && ordered.every(Boolean) ? ordered : null
+  }
+
+  /**
+   * Lists the archive whose first volume starts with `firstData` (a decoded
+   * first segment). Null when it isn't a 7z start, when volumes are missing,
+   * or when the tail doesn't hold the header its start points to.
+   */
+  async list(firstFile: any, firstData: Buffer): Promise<RarInfo | null> {
+    const start = parseSevenZipStart(firstData)
+    if (!start) return null
+    const volumes = await this.volumes(firstFile)
+    if (!volumes) return null
+    if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1)
+
+    const tail = await this.readTail(volumes[volumes.length - 1], start)
+    if (!tail) return null
+
+    const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
+    try {
+      const archive = path.join(dir, 'archive.7z')
+      const handle = await open(archive, 'w')
+      try {
+        await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
+        await handle.write(tail.data, 0, tail.data.length, tail.from)
+        await handle.truncate(start.archiveSize)
+      } finally {
+        await handle.close()
+      }
+      return await this.bsdtar(archive, volumes.length > 1)
+    } finally {
+      await rm(dir, { recursive: true, force: true })
+    }
+  }
+
+  /** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
+  private async bsdtar(archive: string, split: boolean): Promise<RarInfo | null> {
+    let stdout = ''
+    try {
+      stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
+    } catch (error: any) {
+      if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split)
+      logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
+      return null
+    }
+    const entries = parseBsdtarListing(stdout)
+    const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
+    let encrypted = false
+    if (sample) {
+      try {
+        await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
+      } catch (error: any) {
+        // Without the data (it's all zeros here), anything else fails too: only this message counts.
+        encrypted = /content is encrypted/i.test(String(error.stderr))
+      }
+    }
+    return this.info(entries, false, encrypted, split)
+  }
+
+  private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo {
+    return {
+      format: '7z',
+      volume: split,
+      firstVolume: true,
+      solid: false,
+      headersEncrypted,
+      truncated: false,
+      entries: entries.map((e) => ({
+        name: e.name,
+        size: e.size,
+        packedSize: 0,
+        directory: e.directory,
+        encrypted: encrypted && !e.directory,
+        splitBefore: false,
+        splitAfter: false,
+        stored: false,
+        dataOffset: 0,
+      })),
+    }
+  }
+
+  /**
+   * The archive's tail from the last volume: the header (checked against its
+   * CRC) and, for an encoded header, the packed stream before it. `from` is
+   * its position in the whole archive.
+   */
+  private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> {
+    const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
+    const numbers = [...ids.keys()]
+    if (numbers.length === 0) return null
+    const segments = new Map<number, DecodedSegment>()
+    const segment = async (n: number) => {
+      if (!segments.has(n)) {
+        const id = ids.get(n)?.id
+        if (!id) throw new Error(`part ${n} of ${last.filename} is missing`)
+        segments.set(n, decodeSegment(await this.body(id), { lenientSize: true }))
+      }
+      return segments.get(n)!
+    }
+
+    const lastNumber = numbers[numbers.length - 1]
+    const lastSegment = await segment(lastNumber)
+    const volumeSize = lastSegment.meta.begin.size
+    const volumeStart = start.archiveSize - volumeSize
+    const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
+    const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
+    if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
+
+    // Bytes [from, to) of the whole archive, from the last volume's parts.
+    const read = async (from: number, to: number): Promise<Buffer | null> => {
+      if (from < volumeStart) return null
+      const a = from - volumeStart
+      const b = to - volumeStart
+      const pieces: Buffer[] = []
+      for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
+        const s = await segment(n)
+        const offset = (s.meta.part?.begin ?? 1) - 1
+        if (offset !== (n - 1) * partSize) return null
+        pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
+      }
+      const data = Buffer.concat(pieces)
+      return data.length === b - a ? data : null
+    }
+
+    const header = await read(start.headerStart, start.archiveSize)
+    if (!header || !checkHeader(start, header)) return null
+    const from = Math.min(listingStart(start, header), start.headerStart)
+    if (start.archiveSize - from > MAX_TAIL_BYTES) return null
+    const data = await read(from, start.archiveSize)
+    return data ? { from, data } : null
+  }
+
+  private async body(messageId: string): Promise<Buffer> {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+    } catch (error: any) {
+      failed = !/^4\d\d/.test(String(error.message))
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+}
+
+export default new SevenZipService()

app/services/SfvService.ts

@@ -0,0 +1,80 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { parseSfv, sfvReleaseName } from '#lib/content/index'
+
+/** Releases made by name rules alone; an SFV's listing outranks them. */
+const WEAK_SOURCES = ['stem', 'weak-stem', 'subject']
+/** Files further apart than this from the SFV in posting time aren't taken. */
+const WINDOW_MS = 2 * 24 * 60 * 60 * 1000
+
+/**
+ * SFV grouping (CONCEPT.md D34). An uploader that posts every track of an
+ * album as its own file (13.5k such releases from one poster on homelab)
+ * still posts the album's SFV, which names them all.
+ */
+class SfvService {
+  /**
+   * Records an SFV file's listing on its release, moves the listed files that
+   * sit in weak releases into it, and names the release after the album when
+   * it's still named after a track or the SFV itself. Returns how many files
+   * moved, or null when it isn't an SFV.
+   */
+  async attach(sfvFile: any, text: string): Promise<number | null> {
+    const entries = parseSfv(text)
+    if (entries.length === 0 || !sfvFile.release_id) return null
+    const releaseId = Number(sfvFile.release_id)
+    const postedAt = sfvFile.date ? new Date(Number(sfvFile.date)) : null
+    await db
+      .knexQuery()
+      .table('sfv_entries')
+      .insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
+      .onConflict(['release_id', 'name'])
+      .ignore()
+    await db.from('files').where('id', sfvFile.id).update({ content_type: 'sfv' })
+
+    const candidates = await db
+      .from('files as f')
+      .join('releases as r', 'r.id', 'f.release_id')
+      .whereIn(db.raw('coalesce(f.real_name, f.filename)') as any, entries.map((e) => e.name))
+      .whereNot('f.release_id', releaseId)
+      .whereIn('r.source', WEAK_SOURCES)
+      .select('f.id', 'f.poster', 'f.date')
+    const moving = candidates.filter((f) => this.sameUpload(f, sfvFile.poster, postedAt))
+    await this.move(moving.map((f) => Number(f.id)), releaseId)
+
+    const release = await db.from('releases').where('id', releaseId).first()
+    const album = sfvReleaseName(String(sfvFile.real_name ?? sfvFile.filename))
+    if (release && WEAK_SOURCES.includes(release.source) && album !== release.name && /^\d{1,3}-/.test(release.name)) {
+      await db.from('releases').where('id', releaseId).update({ name: album })
+    }
+    logger.info(`SFV "${sfvFile.filename}": ${entries.length} files listed, ${moving.length} moved into release ${releaseId} "${album}".`)
+    return moving.length
+  }
+
+  /** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
+  async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
+    const entries = await db.from('sfv_entries').where('name', file.name)
+    const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
+    if (!entry || Number(entry.release_id) === Number(file.release_id)) return null
+    if (file.release_id) {
+      const current = await db.from('releases').where('id', file.release_id).first()
+      if (current && !WEAK_SOURCES.includes(current.source)) return null
+    }
+    await this.move([file.id], Number(entry.release_id))
+    return Number(entry.release_id)
+  }
+
+  private sameUpload(file: { poster?: string | null; date?: number | string | null }, poster: string | null, postedAt: Date | null) {
+    if (poster && file.poster && file.poster !== poster) return false
+    if (postedAt && file.date && Math.abs(Number(file.date) - postedAt.getTime()) > WINDOW_MS) return false
+    return true
+  }
+
+  private async move(fileIds: number[], releaseId: number) {
+    if (fileIds.length === 0) return
+    await db.from('files').whereIn('id', fileIds).update({ release_id: releaseId })
+    await db.from('binaries').whereIn('file_id', fileIds).update({ release_id: releaseId })
+  }
+}
+
+export default new SfvService()

app/services/Shutdown.ts

@@ -0,0 +1,45 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+
+/** After a signal, give cleanup this long before exiting anyway. */
+const FORCE_EXIT_MS = 25_000
+
+/**
+ * Graceful shutdown for the long-running commands. SIGTERM (docker stop) and
+ * SIGINT wake up `wait()` and `sleep()`, so workers can finish their active
+ * jobs and close their connections before the process exits.
+ */
+class Shutdown {
+  private readonly controller = new AbortController()
+
+  constructor() {
+    for (const signal of ['SIGTERM', 'SIGINT'] as const) {
+      process.once(signal, () => {
+        this.controller.abort()
+        setTimeout(() => process.exit(0), FORCE_EXIT_MS).unref()
+      })
+    }
+  }
+
+  get requested(): boolean {
+    return this.controller.signal.aborted
+  }
+
+  /** Resolves when a shutdown signal arrives. */
+  wait(): Promise<void> {
+    if (this.requested) return Promise.resolve()
+    return new Promise((resolve) =>
+      this.controller.signal.addEventListener('abort', () => resolve(), { once: true })
+    )
+  }
+
+  /** Sleeps, but returns early on shutdown. */
+  async sleep(ms: number): Promise<void> {
+    try {
+      await sleep(ms, undefined, { signal: this.controller.signal })
+    } catch {
+      // aborted by a shutdown signal
+    }
+  }
+}
+
+export default new Shutdown()

app/services/SplitService.ts

@@ -0,0 +1,146 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import IngestService from '#services/IngestService'
+import NameService from '#services/NameService'
+import { splitVerified } from '#lib/split/index'
+
+/** Conflicted binaries handled per run; the rest follow on the next tick. */
+const BATCH = 200
+
+export interface SplitResult {
+  parents: number
+  volumes: number
+  completed: number
+}
+
+/**
+ * Splits conflicted obfuscated binaries (several equal-sized files under one
+ * size+parts identity) into one binary per volume. Only volumes whose
+ * combined part CRCs match the file CRC of their last part are created, so a
+ * split can never produce a corrupt file; the rest waits for more articles.
+ * See CONCEPT.md (D20).
+ */
+class SplitService {
+  async splitPending(): Promise<SplitResult> {
+    const parents = await db
+      .from('binaries')
+      .whereNotNull('conflicted_at')
+      .whereNull('parent_binary_id')
+      // split_at = 'infinity' marks binaries that cannot be split (see migration …810).
+      .where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
+      // Most recently active first: those are the posts still coming in.
+      .orderBy('last_seen_at', 'desc')
+      .limit(BATCH)
+
+    const volumeIds: number[] = []
+    for (const parent of parents) {
+      volumeIds.push(...(await this.split(parent)))
+    }
+
+    // Former conflicts that turned out to be reposts (unflagged, keep their
+    // split_at): no new article may arrive to trigger their completion.
+    const unflagged = await db
+      .from('binaries')
+      .whereNull('conflicted_at')
+      .whereNull('completed_at')
+      .whereNull('parent_binary_id')
+      .whereNotNull('split_at')
+      .whereRaw("split_at <> 'infinity'")
+      .limit(BATCH)
+      .select('id')
+    volumeIds.push(...unflagged.map((b) => Number(b.id)))
+
+    await IngestService.matchPar2Releases(volumeIds)
+    await NameService.resolve(volumeIds)
+    const completed = await IngestService.claimCompleted(volumeIds)
+    await IngestService.enqueue(completed)
+    return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
+  }
+
+  /** Creates the newly verified volumes of one conflicted binary; returns their ids. */
+  private async split(parent: any): Promise<number[]> {
+    return db.transaction(async (trx) => {
+      // Unverified volumes from before verification existed can never complete.
+      await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()
+
+      const { rows } = await trx.rawQuery(
+        `WITH taken AS (
+           SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
+           WHERE v.parent_binary_id = ?
+         ), candidates AS (
+           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
+           UNION ALL
+           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
+         )
+         SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
+        [parent.id, parent.id, parent.id]
+      )
+      const volumes = splitVerified(
+        rows.map((r: any) => ({
+          number: r.number,
+          messageId: r.message_id,
+          bytes: r.bytes,
+          articleNumber: r.article_number === null ? null : Number(r.article_number),
+          pcrc32: r.pcrc32,
+          partSize: r.part_size,
+          fileCrc32: r.file_crc32,
+        })),
+        { totalParts: parent.total_parts }
+      )
+
+      const now = new Date()
+      const ids: number[] = []
+      for (const articles of volumes) {
+        // A verified volume is fixed, so its first article identifies it.
+        const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
+        const inserted = await trx
+          .knexClient('binaries')
+          .insert({
+            file_key: fileKey,
+            poster: parent.poster,
+            subject: parent.subject,
+            name: parent.name,
+            total_parts: parent.total_parts,
+            groups: JSON.stringify(parent.groups),
+            posted_at: parent.posted_at,
+            size: parent.size,
+            release_id: parent.release_id,
+            parent_binary_id: parent.id,
+            first_seen_at: now,
+            last_seen_at: now,
+          })
+          .onConflict('file_key')
+          .ignore()
+          .returning('id')
+        if (inserted.length === 0) continue
+        const id = Number(inserted[0].id)
+        await trx.knexClient('parts').insert(
+          articles.map((a) => ({
+            binary_id: id,
+            number: a.number,
+            message_id: a.messageId,
+            bytes: a.bytes,
+            article_number: a.articleNumber,
+            pcrc32: a.pcrc32,
+            part_size: a.partSize,
+            file_crc32: a.fileCrc32,
+          }))
+        )
+        // The fingerprint of a volume is that of its part 1 article.
+        await trx.rawQuery(
+          `UPDATE binaries SET md5_16k = (
+             SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
+             UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
+           WHERE id = ?`,
+          [parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
+        )
+        ids.push(id)
+      }
+
+      await trx.from('binaries').where('id', parent.id).update({ split_at: now })
+      return ids
+    })
+  }
+}
+
+export default new SplitService()

app/services/SpotService.ts

@@ -0,0 +1,389 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import NzbImportService from '#services/NzbImportService'
+import IngestService, { type OverviewHeader } from '#services/IngestService'
+import { parseNzb } from '#lib/nzb/index'
+import {
+  moderationTarget,
+  passwordFromSpot,
+  spotterId,
+  decodeSpotBinary,
+  decodeSpotText,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  verifySpotHeader,
+  type SpotHeader,
+  type SpotXml,
+} from '#lib/spotnet/index'
+
+/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
+const MAX_NZB_SEGMENTS = 20
+/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
+const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
+/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
+const INSERT_CHUNK = 2000
+/** Spot images larger than this many articles aren't fetched. */
+const MAX_IMAGE_SEGMENTS = 10
+/** A claimed spot whose worker died is taken again after this long. */
+const RECLAIM_AFTER = '1 hour'
+
+/**
+ * A spot's title: from its XML, which is decoded properly (the Subject of a
+ * raw header is read as ISO-8859-1), else from the header.
+ */
+function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
+  return head.xml?.title || head.header.title
+}
+
+/**
+ * A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
+ * spots): less than 1% of the size the spot announces (the real NZB is
+ * handed out elsewhere), or under 16 KB in all (text-only spots: requests,
+ * announcements). Its files are not imported, and the spot is hidden.
+ */
+export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
+  const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
+  return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
+}
+
+export function isSpotGroup(name: string): boolean {
+  return indexerConfig.spotGroups.includes(name)
+}
+
+/**
+ * Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
+ * header holds XML with the message ids of a compressed NZB, which is
+ * imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
+ * (nyuu, ngPost, the skipped multi-group uploader).
+ */
+class SpotService {
+  /** Queues the spots among a batch of one spot group's headers. Returns how many. */
+  async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
+    const now = new Date()
+    const rows = []
+    for (const [articleNumber, header] of overviews) {
+      if (!header?.from || !header['message-id']) continue
+      const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
+      if (!spot?.title) continue
+      rows.push({
+        message_id: spot.messageId,
+        group_name: groupName,
+        article_number: Number(articleNumber),
+        title: spot.title,
+        tag: spot.tag || null,
+        poster: spot.poster,
+        category: spot.category,
+        subcats: JSON.stringify(spot.subcats),
+        size: spot.size || null,
+        posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
+        key_id: spot.keyId,
+        spotter_id: spotterId(spot.selfSignedPubKey) || null,
+        // From the overview; titles with accents may fail here (UTF-8 decoding)
+        // and are verified again on their raw header by worker:spots.
+        verified: verifySpotHeader(spot),
+        created_at: now,
+      })
+    }
+    for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
+      await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
+    }
+    return rows.length
+  }
+
+  /**
+   * Takes up to `limit` pending spots, marked `processing`: recent ones
+   * (to be imported) first, then the history in posting order, so live spots
+   * don't wait behind a backfill and moderation follows its targets.
+   */
+  async claim(limit: number): Promise<any[]> {
+    const claim = async (filter: string, bindings: unknown[], n: number) =>
+      (
+        await db.rawQuery(
+          `UPDATE spots SET status = 'processing', processed_at = now()
+           WHERE message_id IN (
+             SELECT message_id FROM spots WHERE ${filter}
+             ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
+           RETURNING *`,
+          [...bindings, n]
+        )
+      ).rows as any[]
+    const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
+    if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
+    if (rows.length < limit) {
+      rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
+    }
+    return rows
+  }
+
+  /** Spots posted before this are listed, not imported: their NZB is read on first download. */
+  importCutoff(): Date {
+    return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
+  }
+
+  /** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
+  isListable(spot: any): boolean {
+    return (
+      spot.verified === true &&
+      spot.posted_at !== null &&
+      new Date(spot.posted_at) < this.importCutoff() &&
+      !moderationTarget({ keyId: spot.key_id, title: spot.title })
+    )
+  }
+
+  /**
+   * Creates the listing releases (source `spot`, no files) for these spots.
+   * The release key is the one an import of the spot's NZB uses, so the
+   * import on first download fills this release.
+   */
+  async list(spots: any[]): Promise<number> {
+    if (spots.length === 0) return 0
+    const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
+    const ids = await IngestService.upsertReleases(
+      db.connection().getWriteClient(),
+      spots.map((spot) => ({
+        release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
+        poster: spot.poster,
+        groups: [spot.group_name],
+        postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
+      }))
+    )
+    const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
+    await db.rawQuery(
+      `UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
+       WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
+      [JSON.stringify(rows)]
+    )
+    await db.rawQuery(
+      `UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
+       FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
+       WHERE s.message_id = v.message_id`,
+      [JSON.stringify(rows)]
+    )
+    return rows.length
+  }
+
+  /**
+   * Verifies the spot on its raw header (the signature covers ISO-8859-1
+   * bytes, which the overview, read as UTF-8, may have mangled), then
+   * applies moderation, lists an old spot or imports a recent one's NZB.
+   * Returns the new status.
+   */
+  async process(spot: any): Promise<string> {
+    const update: Record<string, unknown> = { processed_at: new Date(), error: null }
+    try {
+      const head = await this.readHead(spot.message_id)
+      if (!head) {
+        update.status = 'unverified'
+      } else if (moderationTarget(head.header)) {
+        update.status = 'moderation'
+        await this.moderate(head.header, moderationTarget(head.header)!)
+      } else {
+        update.title = spotTitle(head)
+        update.poster = head.header.poster
+        update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
+        Object.assign(update, this.details(head.xml))
+        if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
+          await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
+          update.status = 'listed'
+        } else {
+          Object.assign(update, await this.importNzb(spot.message_id, head))
+        }
+      }
+    } catch (error: any) {
+      update.status = 'failed'
+      update.error = String(error.message).slice(0, 500)
+      logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
+    }
+    await db.from('spots').where('message_id', spot.message_id).update(update)
+    if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
+    return update.status as string
+  }
+
+  /**
+   * The NZB of a listing, on first download: reads and imports it. Returns
+   * the release that now holds its files (the listing's own, unless the NZB
+   * joined files already known elsewhere), or null.
+   */
+  async importListing(releaseId: number): Promise<number | null> {
+    const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
+    if (!spot) return null
+    return this.reimport(spot)
+  }
+
+  /**
+   * Reads a spot's NZB again and imports it, so its files end up in the
+   * spot's own release under the spot's title. Returns that release, or null.
+   */
+  async reimport(spot: any): Promise<number | null> {
+    const head = await this.readHead(spot.message_id)
+    if (!head) return null
+    const result = await this.importNzb(spot.message_id, head)
+    await db
+      .from('spots')
+      .where('message_id', spot.message_id)
+      .update({ ...result, ...this.details(head.xml), processed_at: new Date() })
+    await this.applyPassword(spot.message_id)
+    return (result.release_id as number | undefined) ?? null
+  }
+
+  /**
+   * A password the spotter wrote in the spot's title or description (not in
+   * the NZB) goes on its release, unless the release has one already.
+   */
+  async applyPassword(messageId: string): Promise<string | null> {
+    const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
+    const password = spot ? passwordFromSpot(spot.title, spot.description) : null
+    if (!password) return null
+    await db
+      .from('releases')
+      .where('id', spot.release_id)
+      .whereNull('password')
+      .update({ password, password_source: 'spot' })
+    return password
+  }
+
+  /** Columns for what the spot's XML adds to its overview: description, image, website. */
+  private details(xml: SpotXml | null): Record<string, unknown> {
+    return {
+      description: xml?.description || null,
+      image: xml?.image ? JSON.stringify(xml.image) : null,
+      ...(xml?.website ? { website: xml.website } : {}),
+      details_at: new Date(),
+    }
+  }
+
+  /**
+   * A spot row with its description and image, reading its full header
+   * first when that hasn't happened yet (old spots are listed from their
+   * overview only). Returns the row as it is when Usenet can't be reached.
+   */
+  async withDetails(spot: any): Promise<any> {
+    if (spot.details_at) return spot
+    try {
+      const head = await this.readHead(spot.message_id)
+      const update = head ? this.details(head.xml) : { details_at: new Date() }
+      await db.from('spots').where('message_id', spot.message_id).update(update)
+      await this.applyPassword(spot.message_id)
+      return { ...spot, ...update, image: head?.xml?.image ?? null }
+    } catch (error: any) {
+      logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
+      // The article is gone (430): nothing to read later either.
+      if (/^4\d\d/.test(String(error.message))) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+      }
+      return spot
+    }
+  }
+
+  /**
+   * Reads the details of up to `limit` imported spots that don't have them
+   * (imported before details were kept), newest first. Returns how many.
+   * Listed spots are left to their first view: there are millions.
+   * Returns how many were read (0 when Usenet can't be reached).
+   */
+  async fillDetails(limit: number): Promise<number> {
+    const spots = await db
+      .from('spots')
+      .where('status', 'imported')
+      .whereNull('details_at')
+      .orderBy('posted_at', 'desc')
+      .limit(limit)
+    let read = 0
+    for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
+    return read
+  }
+
+  /** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
+  async image(spot: any): Promise<Buffer | null> {
+    const segments: string[] = spot.image?.segments ?? []
+    if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
+    const bodies = []
+    for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+    return decodeSpotBinary(bodies, false)
+  }
+
+  /** The spot's raw header, verified; null when it doesn't verify. */
+  private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
+    const raw = await this.fetch(`HEAD <${messageId}>`)
+    const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
+    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
+    if (!header || !verifySpotHeader(header)) return null
+    return { header, xml: parseSpotXml(decodeSpotText(xml)) }
+  }
+
+  /** Downloads the spot's compressed NZB and imports it as a release (D26). */
+  private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
+    const segments = head.xml?.nzbSegments ?? []
+    if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
+    const bodies = []
+    for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+    const nzbXml = decodeSpotBinary(bodies).toString('utf8')
+    const nzb = parseNzb(nzbXml)
+    if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
+    const { header } = head
+    if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
+    const title = spotTitle(head)
+    const result = await NzbImportService.import(
+      {
+        id: null,
+        filename: `${title}.nzb`,
+        poster: header.poster,
+        groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
+        date: header.stamp * 1000,
+        first_message_id: messageId,
+      },
+      nzbXml,
+      { name: title, source: 'spot', moveContainer: false, trusted: true }
+    )
+    return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
+  }
+
+  /**
+   * Applies a delete/dispose/remove: the target spot is marked `removed`
+   * (its release is then hidden from search). As in Spotweb, a personal
+   * dispose must come from the same spotter within 5 days; a moderator's
+   * (signed with Spotnet's key, no personal key) always counts.
+   */
+  private async moderate(header: SpotHeader, targetId: string) {
+    const target = await db.from('spots').where('message_id', targetId).first()
+    if (!target) return
+    const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
+    if (personal) {
+      const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
+      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
+    }
+    await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
+    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+    // Spots that joined its release (a correction, a repost) take the files over now.
+    if (target.release_id) {
+      const sharing = await db
+        .from('spots')
+        .where('release_id', target.release_id)
+        .where('status', 'imported')
+        .whereNot('message_id', targetId)
+      for (const spot of sharing) await this.reimport(spot)
+    }
+  }
+
+  /** One HEAD or BODY, as raw bytes. */
+  private async fetch(command: string): Promise<Buffer> {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
+      return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
+    } catch (error: any) {
+      // 430 (no such article) is an answer, not a broken connection.
+      failed = !/^4\d\d/.test(String(error.message))
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+}
+
+export default new SpotService()

app/services/StagingService.ts

@@ -0,0 +1,252 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import IngestService, { type IngestSegment } from '#services/IngestService'
+import { counterFileKey } from '#lib/subject/index'
+import { fileStem } from '#lib/release/index'
+
+/** Rows per statement (a key and a JSON array each). */
+const CHUNK = 5000
+
+/**
+ * Staging of lone header segments (CONCEPT.md D35). Uploads that give every
+ * article its own random subject turned each article into a binary, a part
+ * and a release, which GC deleted two hours later: ~3.9M binaries an hour
+ * against ~123k real files. A segment whose file is new and alone in its
+ * fetch batch now waits in `staged_segments` (unlogged, one index) until a
+ * second segment of that file arrives; only then do both go into the real
+ * tables. GC drops what waited longer than STAGING_TTL_SECONDS (30 min).
+ *
+ * Counter grouping (D36): those random-subject uploads still keep poster,
+ * file counter `[i/M]` and part count per file. Staged segments also carry
+ * that key (`counter_key`); once two staged segments from different subjects
+ * share it, they become one file under that key, and later segments with it
+ * join that file directly.
+ */
+class StagingService {
+  enabled(groupName: string): boolean {
+    const groups = indexerConfig.stagingGroups
+    return groups.includes('*') || groups.includes(groupName)
+  }
+
+  /** Whether counter grouping (D36) applies to this group. */
+  countersEnabled(groupName: string): boolean {
+    const groups = indexerConfig.counterGroupingGroups
+    return groups.includes('*') || groups.includes(groupName)
+  }
+
+  /**
+   * The segments of a header batch that should be ingested now: the batch's
+   * own, minus lone segments of new files (staged instead), plus staged
+   * segments whose file this batch shows to be real. `staged` counts the
+   * segments that went to staging.
+   */
+  async filter(segments: IngestSegment[], { counters = false } = {}): Promise<{ ingest: IngestSegment[]; staged: number }> {
+    const byKey = new Map<string, IngestSegment[]>()
+    for (const segment of segments) {
+      const list = byKey.get(segment.fileKey)
+      if (list) list.push(segment)
+      else byKey.set(segment.fileKey, [segment])
+    }
+
+    const ingest: IngestSegment[] = []
+    const lone: IngestSegment[] = []
+    for (const list of byKey.values()) {
+      // Single-segment files are complete; two different articles of a file in one batch make it real.
+      // (One article twice is a crosspost, LOG.md 2026-10-02.)
+      const articles = new Set(list.map((s) => s.messageId)).size
+      if (list[0].totalParts <= 1 || articles >= 2 || list[0].weakIdentity || list[0].size != null) ingest.push(...list)
+      else lone.push(list[0])
+    }
+
+    const counterOf = new Map<IngestSegment, string>()
+    if (counters) {
+      for (const segment of lone) {
+        const key = counterFileKey(segment)
+        if (key) counterOf.set(segment, key)
+      }
+    }
+
+    // Files we already have a binary for (by subject, or by counters) take their segments directly.
+    const known = new Set<string>()
+    const lookup = [...new Set([...lone.map((s) => s.fileKey), ...counterOf.values()])]
+    for (let i = 0; i < lookup.length; i += CHUNK) {
+      const rows = await db.from('binaries').whereIn('file_key', lookup.slice(i, i + CHUNK)).select('file_key')
+      for (const row of rows) known.add(row.file_key)
+    }
+    const toStage: IngestSegment[] = []
+    const knownCounters = new Set<string>()
+    for (const segment of lone) {
+      const counter = counterOf.get(segment)
+      if (known.has(segment.fileKey)) {
+        ingest.push(segment)
+      } else if (counter && known.has(counter)) {
+        ingest.push(asCounterFile(segment, counter))
+        knownCounters.add(counter)
+      } else {
+        toStage.push(segment)
+      }
+    }
+
+    // Stage the rest; a file that now has two staged segments comes out again.
+    const promoted: string[] = []
+    // Key order, so concurrent jobs lock rows in the same order.
+    toStage.sort((a, b) => a.fileKey.localeCompare(b.fileKey))
+    for (let i = 0; i < toStage.length; i += CHUNK) {
+      const rows = toStage.slice(i, i + CHUNK).map((s) => ({
+        file_key: s.fileKey,
+        counter_key: counterOf.get(s) ?? null,
+        segments: [serialize(s)],
+        staged_at: new Date(),
+      }))
+      const { rows: result } = await retryDeadlock(() => db.rawQuery(
+        `INSERT INTO staged_segments (file_key, counter_key, segments, staged_at)
+         SELECT file_key, counter_key, segments, staged_at
+         FROM jsonb_to_recordset(?::jsonb) AS v(file_key text, counter_key text, segments jsonb, staged_at timestamptz)
+         ON CONFLICT (file_key) DO UPDATE SET segments = CASE
+           -- The same article again (a crosspost, or a re-fetched range) is no second segment.
+           WHEN EXISTS (SELECT 1 FROM jsonb_array_elements(staged_segments.segments) e
+                        WHERE e->>'messageId' = EXCLUDED.segments->0->>'messageId') THEN staged_segments.segments
+           ELSE staged_segments.segments || EXCLUDED.segments END
+         RETURNING file_key, jsonb_array_length(segments) AS n`,
+        [JSON.stringify(rows)]
+      ))
+      for (const row of result) if (Number(row.n) >= 2) promoted.push(row.file_key)
+    }
+
+    // Counter files: two staged rows (two subjects) with one counter key make a file.
+    const stagedCounters = [...new Set(toStage.map((s) => counterOf.get(s)).filter((k): k is string => Boolean(k)))]
+    const readyCounters: string[] = []
+    for (let i = 0; i < stagedCounters.length; i += CHUNK) {
+      const { rows } = await db.rawQuery(
+        `SELECT counter_key FROM staged_segments WHERE counter_key = ANY(?) GROUP BY counter_key HAVING count(*) >= 2`,
+        [stagedCounters.slice(i, i + CHUNK)]
+      )
+      for (const row of rows) readyCounters.push(row.counter_key)
+    }
+
+    // Promoted files, staged segments of files this batch ingests anyway, and ready counter files.
+    const fileKeys = [...new Set([...promoted, ...ingest.map((s) => s.fileKey)])]
+    const counterKeys = [...new Set([...readyCounters, ...knownCounters])]
+    const seen = new Set(ingest.map((s) => s.messageId))
+    for (const segment of await this.take(fileKeys, counterKeys)) {
+      if (!seen.has(segment.messageId)) ingest.push(segment)
+      seen.add(segment.messageId)
+    }
+    const staged = toStage.filter((s) => !seen.has(s.messageId)).length
+    return { ingest, staged }
+  }
+
+  /**
+   * GC: drops staged segments that waited longer than `maxAgeSeconds`, then
+   * ingests those whose file got a binary meanwhile (two fetch jobs racing).
+   * Expiry comes first so the table stays bounded whatever the budget; the
+   * adoption starts from recently seen binaries (index on last_seen_at,
+   * which moves in 10-minute steps) rather than joining every staged row.
+   */
+  async sweep(maxAgeSeconds: number, budget: number): Promise<{ adopted: number; expired: number }> {
+    let adopted = 0
+    let expired = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM staged_segments WHERE ctid IN (
+           SELECT ctid FROM staged_segments WHERE staged_at < now() - make_interval(secs => ?) LIMIT 20000)`,
+        [maxAgeSeconds]
+      )
+      expired += result.rowCount
+      if (result.rowCount < 20000) break
+    }
+    let lastId = 0
+    while (Date.now() < budget) {
+      const recent = await db
+        .from('binaries')
+        .where('last_seen_at', '>', new Date(Date.now() - (maxAgeSeconds + 15 * 60) * 1000))
+        .where('id', '>', lastId)
+        .orderBy('id')
+        .limit(CHUNK)
+        .select('id', 'file_key')
+      if (recent.length === 0) break
+      lastId = Number(recent[recent.length - 1].id)
+      const keys = recent.map((b) => b.file_key as string)
+      // A binary's key is a subject key or a counter key; staged rows are looked up by both.
+      const segments = await this.take(keys, keys)
+      if (segments.length > 0) await IngestService.ingest(segments)
+      adopted += segments.length
+    }
+    return { adopted, expired }
+  }
+
+  /**
+   * Removes and returns the staged segments of these files: by subject key,
+   * or by counter key (returned as segments of the counter file).
+   */
+  private async take(fileKeys: string[], counterKeys: string[] = []): Promise<IngestSegment[]> {
+    const taken: IngestSegment[] = []
+    for (let i = 0; i < fileKeys.length; i += CHUNK) {
+      const { rows } = await retryDeadlock(() =>
+        db.rawQuery(`DELETE FROM staged_segments WHERE file_key = ANY(?) RETURNING segments`, [fileKeys.slice(i, i + CHUNK)])
+      )
+      for (const row of rows) taken.push(...(row.segments as any[]).map(deserialize))
+    }
+    for (let i = 0; i < counterKeys.length; i += CHUNK) {
+      const { rows } = await retryDeadlock(() =>
+        db.rawQuery(`DELETE FROM staged_segments WHERE counter_key = ANY(?) RETURNING counter_key, segments`, [counterKeys.slice(i, i + CHUNK)])
+      )
+      for (const row of rows) taken.push(...(row.segments as any[]).map((s) => asCounterFile(deserialize(s), row.counter_key)))
+    }
+    return taken
+  }
+}
+
+/**
+ * A segment as part of its counter file (D36): the counter key is its file
+ * key, it is a weak identity (a repeated part number from another upload
+ * marks the binary conflicted), and its release is a weak one of its own,
+ * which the upload's par2 set can gather later.
+ */
+function asCounterFile(segment: IngestSegment, counterKey: string): IngestSegment {
+  return {
+    ...segment,
+    fileKey: counterKey,
+    weakIdentity: true,
+    release: {
+      key: createHash('sha1').update(`counter-release\n${counterKey}`).digest('hex'),
+      name: fileStem(segment.name),
+      source: 'weak-stem',
+    },
+  }
+}
+
+/**
+ * Runs a staging statement again when it lost a deadlock: concurrent fetch
+ * jobs insert by file key and delete by counter key, so their row locks can
+ * cross. Postgres rolled the statement back, and each one is safe to repeat.
+ */
+async function retryDeadlock<T>(statement: () => Promise<T>, attempts = 5): Promise<T> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await statement()
+    } catch (error: any) {
+      if (error?.code !== '40P01' || attempt >= attempts) throw error
+      await new Promise((resolve) => setTimeout(resolve, 50 * attempt + Math.random() * 100))
+    }
+  }
+}
+
+function serialize(segment: IngestSegment) {
+  return {
+    ...segment,
+    postedAt: segment.postedAt?.toISOString(),
+    articleNumber: segment.articleNumber?.toString(),
+  }
+}
+
+function deserialize(stored: any): IngestSegment {
+  return {
+    ...stored,
+    postedAt: stored.postedAt ? new Date(stored.postedAt) : undefined,
+    articleNumber: stored.articleNumber ?? undefined,
+  }
+}
+
+export default new StagingService()

app/transformers/user_transformer.ts

@@ -0,0 +1,15 @@
+import type User from '#models/user'
+import { BaseTransformer } from '@adonisjs/core/transformers'
+
+export default class UserTransformer extends BaseTransformer<User> {
+  toObject() {
+    return this.pick(this.resource, [
+      'id',
+      'fullName',
+      'email',
+      'createdAt',
+      'updatedAt',
+      'initials',
+    ])
+  }
+}

app/validators/user.ts

@@ -0,0 +1,26 @@
+import vine from '@vinejs/vine'
+
+/**
+ * Shared rules for email and password.
+ */
+const email = () => vine.string().email().maxLength(254)
+const password = () => vine.string().minLength(8).maxLength(32)
+
+/**
+ * Validator to use when performing self-signup
+ */
+export const signupValidator = vine.create({
+  fullName: vine.string().nullable(),
+  email: email().unique({ table: 'users', column: 'email' }),
+  password: password(),
+  passwordConfirmation: password().sameAs('password'),
+})
+
+/**
+ * Validator to use before validating user credentials
+ * during login
+ */
+export const loginValidator = vine.create({
+  email: email(),
+  password: vine.string(),
+})

bin/console.ts

@@ -0,0 +1,47 @@
+/*
+|--------------------------------------------------------------------------
+| Ace entry point
+|--------------------------------------------------------------------------
+|
+| The "console.ts" file is the entrypoint for booting the AdonisJS
+| command-line framework and executing commands.
+|
+| Commands do not boot the application, unless the currently running command
+| has "options.startApp" flag set to true.
+|
+*/
+
+await import('reflect-metadata')
+const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
+
+/**
+ * URL to the application root. AdonisJS need it to resolve
+ * paths to file and directories for scaffolding commands
+ */
+const APP_ROOT = new URL('../', import.meta.url)
+
+/**
+ * The importer is used to import files in context of the
+ * application.
+ */
+const IMPORTER = (filePath: string) => {
+  if (filePath.startsWith('./') || filePath.startsWith('../')) {
+    return import(new URL(filePath, APP_ROOT).href)
+  }
+  return import(filePath)
+}
+
+new Ignitor(APP_ROOT, { importer: IMPORTER })
+  .tap((app) => {
+    app.booting(async () => {
+      await import('#start/env')
+    })
+    app.listen('SIGTERM', () => app.terminate())
+    app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate())
+  })
+  .ace()
+  .handle(process.argv.splice(2))
+  .catch((error) => {
+    process.exitCode = 1
+    prettyPrintError(error)
+  })

bin/server.ts

@@ -0,0 +1,45 @@
+/*
+|--------------------------------------------------------------------------
+| HTTP server entrypoint
+|--------------------------------------------------------------------------
+|
+| The "server.ts" file is the entrypoint for starting the AdonisJS HTTP
+| server. Either you can run this file directly or use the "serve"
+| command to run this file and monitor file changes
+|
+*/
+
+await import('reflect-metadata')
+const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
+
+/**
+ * URL to the application root. AdonisJS need it to resolve
+ * paths to file and directories for scaffolding commands
+ */
+const APP_ROOT = new URL('../', import.meta.url)
+
+/**
+ * The importer is used to import files in context of the
+ * application.
+ */
+const IMPORTER = (filePath: string) => {
+  if (filePath.startsWith('./') || filePath.startsWith('../')) {
+    return import(new URL(filePath, APP_ROOT).href)
+  }
+  return import(filePath)
+}
+
+new Ignitor(APP_ROOT, { importer: IMPORTER })
+  .tap((app) => {
+    app.booting(async () => {
+      await import('#start/env')
+    })
+    app.listen('SIGTERM', () => app.terminate())
+    app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate())
+  })
+  .httpServer()
+  .start()
+  .catch((error) => {
+    process.exitCode = 1
+    prettyPrintError(error)
+  })

bin/test.ts

@@ -0,0 +1,62 @@
+/*
+|--------------------------------------------------------------------------
+| Test runner entrypoint
+|--------------------------------------------------------------------------
+|
+| The "test.ts" file is the entrypoint for running tests using Japa.
+|
+| Either you can run this file directly or use the "test"
+| command to run this file and monitor file changes.
+|
+*/
+
+process.env.NODE_ENV = 'test'
+
+import 'reflect-metadata'
+import { Ignitor, prettyPrintError } from '@adonisjs/core'
+import { configure, processCLIArgs, run } from '@japa/runner'
+
+/**
+ * URL to the application root. AdonisJS need it to resolve
+ * paths to file and directories for scaffolding commands
+ */
+const APP_ROOT = new URL('../', import.meta.url)
+
+/**
+ * The importer is used to import files in context of the
+ * application.
+ */
+const IMPORTER = (filePath: string) => {
+  if (filePath.startsWith('./') || filePath.startsWith('../')) {
+    return import(new URL(filePath, APP_ROOT).href)
+  }
+  return import(filePath)
+}
+
+new Ignitor(APP_ROOT, { importer: IMPORTER })
+  .tap((app) => {
+    app.booting(async () => {
+      await import('#start/env')
+    })
+    app.listen('SIGTERM', () => app.terminate())
+    app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate())
+  })
+  .testRunner()
+  .configure(async (app) => {
+    const { runnerHooks, ...config } = await import('../tests/bootstrap.js')
+
+    processCLIArgs(process.argv.splice(2))
+    configure({
+      ...app.rcFile.tests,
+      ...config,
+      ...{
+        setup: runnerHooks.setup,
+        teardown: runnerHooks.teardown.concat([() => app.terminate()]),
+      },
+    })
+  })
+  .run(() => run())
+  .catch((error) => {
+    process.exitCode = 1
+    prettyPrintError(error)
+  })

commands/BodyQueuePrune.ts

@@ -0,0 +1,97 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import indexerConfig from '#config/indexer'
+import QueueService from '#services/QueueService'
+import RedisService from '#services/RedisService'
+import DeferredService, { isDeferred } from '#services/DeferredService'
+
+const BATCH = 2000
+
+/** Rebuilds the wait list (KEYS[1]) without the ids in the set KEYS[2], in one atomic pass. */
+const FILTER_LIST = `
+local ids = redis.call('LRANGE', KEYS[1], 0, -1)
+local keep = {}
+for _, id in ipairs(ids) do
+  if redis.call('SISMEMBER', KEYS[2], id) == 0 then keep[#keep + 1] = id end
+end
+redis.call('DEL', KEYS[1])
+for i = 1, #keep, 5000 do
+  redis.call('RPUSH', KEYS[1], unpack(keep, i, math.min(i + 4999, #keep)))
+end
+return #ids - #keep`
+
+/**
+ * Removes waiting body jobs whose message-id is on the skip or defer list,
+ * e.g. after adding a pattern (CONCEPT.md D24, D28). Deferred ones are
+ * recorded in deferred_files first, so they can still be processed later.
+ */
+export default class BodyQueuePrune extends BaseCommand {
+  public static commandName = 'body-queue:prune'
+  public static description = 'Drops waiting body jobs that are now skipped or deferred.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count' })
+  declare dryRun: boolean
+
+  @flags.boolean({ description: 'Drop deferred jobs without recording them (already recorded by an earlier run)' })
+  declare skipRecord: boolean
+
+  public async run() {
+    const redis = RedisService.client
+    const queue = QueueService.bodyQueue
+    const waitKey = queue.toKey('wait')
+    let skipped = 0
+    let deferred = 0
+    let scanned = 0
+    const dropped: string[] = []
+    try {
+      const ids: string[] = await redis.lrange(waitKey, 0, -1)
+      for (let i = 0; i < ids.length; i += BATCH) {
+        const batch = ids.slice(i, i + BATCH)
+        const data: [Error | null, string | null][] = await batch
+          .reduce((pipeline, id) => pipeline.hget(queue.toKey(id), 'data'), redis.pipeline())
+          .exec()
+        const drop: string[] = []
+        const toDefer = new Map<string, { articleNumber: number; header: Record<string, string> }[]>()
+        batch.forEach((id, j) => {
+          const job = data[j][1] ? JSON.parse(data[j][1]!) : null
+          if (!job?.header) return
+          const messageId = job.header['message-id'] ?? ''
+          if (isDeferred(job.header)) {
+            const list = toDefer.get(job.group) ?? []
+            list.push({ articleNumber: Number(job.articleNumber), header: job.header })
+            toDefer.set(job.group, list)
+            drop.push(id)
+            deferred++
+          } else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(messageId))) {
+            drop.push(id)
+            skipped++
+          }
+        })
+        scanned += batch.length
+        if (this.dryRun || drop.length === 0) continue
+        if (!this.skipRecord) for (const [group, articles] of toDefer) await DeferredService.record(group, articles)
+        dropped.push(...drop)
+      }
+      if (dropped.length > 0) {
+        const dropKey = queue.toKey('prune-ids')
+        await redis.del(dropKey)
+        for (let i = 0; i < dropped.length; i += BATCH) await redis.sadd(dropKey, ...dropped.slice(i, i + BATCH))
+        await redis.eval(FILTER_LIST, 2, waitKey, dropKey)
+        await redis.del(dropKey)
+        for (let i = 0; i < dropped.length; i += BATCH) {
+          await redis.unlink(...dropped.slice(i, i + BATCH).map((id) => queue.toKey(id)))
+        }
+      }
+      this.logger.info(
+        `Scanned ${scanned} waiting body jobs: ${this.dryRun ? 'would drop' : 'dropped'} ${deferred} deferred and ${skipped} skipped.`
+      )
+    } finally {
+      await QueueService.closeAll()
+      await RedisService.close()
+    }
+  }
+}

commands/ContentBackfill.ts

@@ -0,0 +1,94 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import QueueService from '#services/QueueService'
+import ContentService from '#services/ContentService'
+import { isArchiveStart } from '#lib/content/index'
+
+const BATCH = 1000
+
+/**
+ * Queues completed files for content inspection (names and groupings from
+ * par2 and RAR headers, CONCEPT.md D21) that were completed before it
+ * existed. Costs one article download per file, done by worker:collection.
+ */
+export default class ContentBackfill extends BaseCommand {
+  public static commandName = 'content:backfill'
+  public static description = 'Queues existing files with random names for content inspection.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Queue at most this many files' })
+  declare limit?: number
+
+  @flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
+  declare all: boolean
+
+  @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
+  declare archiveNzbs: boolean
+
+  @flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
+  declare sevenZip: boolean
+
+  @flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
+  declare archives: boolean
+
+  @flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
+  declare sfv: boolean
+
+  @flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
+  declare nicovideo: boolean
+
+  public async run() {
+    let lastId = 0
+    let queued = 0
+    try {
+      while (!this.limit || queued < this.limit) {
+        // Files never inspected: no fingerprint on their binary and no archive listing.
+        const query = db.from('files as f').where('f.id', '>', lastId)
+        // Files from NZBs (spots) have no binary.
+        if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
+        else query.join('binaries as b', 'b.file_id', 'f.id')
+        if (this.nicovideo) {
+          query.whereNull('f.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
+        } else if (this.sfv) {
+          query.whereNull('f.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
+        } else if (this.archives) {
+          query.whereNull('f.content_type').whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
+        } else if (this.sevenZip) {
+          query.whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
+        } else if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
+        else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
+        const files = await query
+          .orderBy('f.id')
+          .limit(BATCH)
+          .select('f.id', 'f.filename')
+        if (files.length === 0) break
+        lastId = Number(files.at(-1).id)
+
+        const jobs = files
+          .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
+          .slice(0, this.limit ? this.limit - queued : undefined)
+          .map((f) => ({
+            name: 'process-collection',
+            data: { fileId: Number(f.id) },
+            // A new id per run: BullMQ ignores an add() for an id it still knows.
+            // Targeted backfills go ahead of whatever is waiting.
+            opts: {
+              jobId: `content-${f.id}-${Date.now()}`,
+              lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
+              // niconico captures are many (~190k) and only cosmetic: behind everything else.
+              ...(this.nicovideo ? { priority: 10 } : {}),
+            },
+          }))
+        if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
+        queued += jobs.length
+      }
+      this.logger.info(`Queued ${queued} files for content inspection.`)
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+}

commands/DbReindex.ts

@@ -0,0 +1,22 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import IndexMaintenance from '#services/IndexMaintenance'
+
+/** Rebuilds bloated indexes online (gc:scheduler also does this weekly). */
+export default class DbReindex extends BaseCommand {
+  public static commandName = 'db:reindex'
+  public static description = 'Rebuilds B-tree indexes that are mostly empty space (online, REINDEX CONCURRENTLY).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show which indexes would be rebuilt' })
+  declare dryRun: boolean
+
+  public async run() {
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`${rebuilt.length} indexes ${this.dryRun ? 'would be rebuilt' : `rebuilt, ${Math.round(saved / 1024 / 1024)} MB freed`}.`)
+  }
+}

commands/DeferredExpandRanges.ts

@@ -0,0 +1,32 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import DeferredService from '#services/DeferredService'
+
+/**
+ * Sends recorded deferred ranges (fully obfuscated ngPost articles, CONCEPT.md
+ * D28) to the body queue, oldest first. Every article in them is downloaded:
+ * their headers can't be grouped into files.
+ */
+export default class DeferredExpandRanges extends BaseCommand {
+  public static commandName = 'deferred:expand-ranges'
+  public static description = 'Queues the bodies of deferred article ranges (costly: every article is downloaded).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Ranges to expand', default: 10 })
+  declare limit: number
+
+  public async run() {
+    try {
+      const { ranges, articles } = await DeferredService.expandRanges(this.limit)
+      this.logger.info(`Expanded ${ranges} ranges: ${articles} articles sent to the body queue.`)
+    } finally {
+      await NntpService.shutdown()
+      await QueueService.closeAll()
+    }
+  }
+}

commands/FilesCleanNames.ts

@@ -0,0 +1,35 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { unquotedFileName } from '#services/NzbImportService'
+
+/**
+ * Files named after a whole NZB subject without quotes (spot NZBs), e.g.
+ * `Adventure.part0055.rar - [056/175] (1/1)`: renamed to the filename in it.
+ */
+export default class FilesCleanNames extends BaseCommand {
+  public static commandName = 'files:clean-names'
+  public static description = 'Strips subject leftovers (counters, "yEnc") from file names that came from NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show what would change' })
+  declare dryRun: boolean
+
+  public async run() {
+    const files = await db
+      .from('files')
+      .whereRaw(`filename ~* '(\\(\\d+/\\d+\\)|yEnc|\\[\\d+/\\d+\\])\\s*$'`)
+      .select('id', 'filename')
+    const changes = files
+      .map((f) => ({ id: Number(f.id), from: f.filename as string, to: unquotedFileName(f.filename) }))
+      .filter((c) => c.to !== c.from)
+    for (const c of changes.slice(0, 5)) this.logger.info(`"${c.from}" → "${c.to}"`)
+    if (!this.dryRun) {
+      for (const c of changes) await db.from('files').where('id', c.id).update({ filename: c.to })
+    }
+    this.logger.info(`${changes.length} of ${files.length} file names ${this.dryRun ? 'would change' : 'cleaned'}.`)
+  }
+}

commands/GcScheduler.ts

@@ -0,0 +1,333 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import type { Queue } from 'bullmq'
+import db from '@adonisjs/lucid/services/db'
+import gcConfig from '#config/gc'
+import QueueService from '#services/QueueService'
+import RedisService from '#services/RedisService'
+import IndexMaintenance from '#services/IndexMaintenance'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+import StagingService from '#services/StagingService'
+import Shutdown from '#services/Shutdown'
+
+const CLEAN_BATCH = 1000
+const SCAN_COUNT = 1000
+
+export default class GcScheduler extends BaseCommand {
+  public static commandName = 'gc:scheduler'
+  public static description = 'Periodically removes old BullMQ jobs and stale binaries.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Run one sweep and exit' })
+  declare once: boolean
+
+  @flags.boolean({ description: 'Report what would be removed without changing anything' })
+  declare dryRun: boolean
+
+  public async run() {
+    this.logger.info(
+      `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
+    )
+
+    try {
+      do {
+        try {
+          await this.sweep()
+        } catch (err: any) {
+          this.logger.error(`GC sweep failed: ${err.message}`)
+        }
+        if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
+      } while (!this.once && !Shutdown.requested)
+    } finally {
+      await QueueService.closeAll()
+      await RedisService.close()
+    }
+  }
+
+  private async sweep() {
+    const { completedAgeSeconds, failedAgeSeconds } = gcConfig.jobs
+    for (const queue of [...QueueService.all, ...QueueService.legacy]) {
+      const completed = await this.cleanJobs(queue, 'completed', completedAgeSeconds)
+      const failed = await this.cleanJobs(queue, 'failed', failedAgeSeconds)
+      if (completed || failed) {
+        this.logger.info(
+          `${queue.name}: ${this.verb('remove', 'removed')} ${completed} completed and ${failed} failed jobs.`
+        )
+      }
+    }
+
+    const trimmed = await this.trimBodyQueue()
+    if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)
+
+    const binaries = await this.deleteStaleBinaries()
+    this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
+
+    if (!this.dryRun) {
+      // Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
+      const budget = () => Date.now() + gcConfig.deleteBudgetMs
+      const parts = await this.dropCopiedParts(budget())
+      const single = await this.deleteSingleArticleBinaries(budget())
+      const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
+      const releases = await this.deleteEmptyReleases(budget())
+      const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
+      if (staging.adopted || staging.expired) {
+        this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
+      }
+      if (parts || single || unfingerprinted || releases) {
+        this.logger.info(
+          `Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
+            `${releases} empty releases deleted.`
+        )
+      }
+    }
+
+    const deferred = await this.deleteOldDeferredFiles()
+    if (deferred) this.logger.info(`Deferred files: ${this.verb('delete', 'deleted')} ${deferred} past retention.`)
+
+    const { expired, scanned } = await this.expireLegacyPendingFiles()
+    if (scanned > 0) {
+      this.logger.info(
+        `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
+      )
+    }
+
+    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep.
+    const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+
+    await this.rebuildIndexes()
+  }
+
+  /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
+  private async rebuildIndexes() {
+    const key = 'gc:reindex:last'
+    const last = Number((await RedisService.client.get(key)) ?? 0)
+    if (Date.now() - last < gcConfig.reindexIntervalMs) return
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
+  }
+
+  /**
+   * Drops the oldest waiting body jobs above gcConfig.bodyQueueMax. BullMQ
+   * workers take jobs from the right end of the wait list, so the oldest are
+   * there.
+   */
+  private async trimBodyQueue(): Promise<number> {
+    const redis = RedisService.client
+    const key = QueueService.bodyQueue.toKey('wait')
+    const excess = (await redis.llen(key)) - gcConfig.bodyQueueMax
+    if (excess <= 0 || this.dryRun) return Math.max(excess, 0)
+    let dropped = 0
+    while (dropped < excess) {
+      const ids: string[] = (await redis.rpop(key, Math.min(5000, excess - dropped))) ?? []
+      if (ids.length === 0) break
+      await redis.unlink(...ids.map((id) => QueueService.bodyQueue.toKey(id)))
+      dropped += ids.length
+    }
+    return dropped
+  }
+
+  /**
+   * Deletes binaries (and, by cascade, their parts) that received no segment
+   * for `staleAfterSeconds`: incomplete ones are given up on; completed ones
+   * already live on in `files`; conflicted ones will never become a file.
+   * Completed ones still waiting for worker:file are kept.
+   */
+  private async deleteStaleBinaries(): Promise<number> {
+    const query = db
+      .from('binaries')
+      .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
+      .where((q) =>
+        q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
+      )
+
+    if (this.dryRun) {
+      const [{ count }] = await query.count('* as count')
+      return Number(count)
+    }
+    return Number(await query.delete())
+  }
+
+  private async deleteOldDeferredFiles(): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.deferredRetentionDays * 24 * 60 * 60 * 1000)
+    let total = 0
+    for (const [table, column] of [['deferred_files', 'first_seen_at'], ['deferred_ranges', 'created_at']]) {
+      const query = db.from(table).where(column, '<', cutoff)
+      if (this.dryRun) {
+        const [{ count }] = await query.count('* as count')
+        total += Number(count)
+      } else {
+        total += Number(await query.delete())
+      }
+    }
+    return total
+  }
+
+  // Keyset cursors over the tables, kept between sweeps so one pass over
+  // millions of rows is spread over several sweeps.
+  private singleCursor = 0
+  private releaseCursor = 0
+
+  /** Parts of header-derived binaries that are a file already (worker:file now drops them itself). */
+  private async dropCopiedParts(budget: number): Promise<number> {
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        // Header-derived parts (no per-part CRC) of binaries that are a file now.
+        `DELETE FROM parts WHERE pcrc32 IS NULL AND binary_id IN (
+           SELECT b.id FROM binaries b
+           WHERE b.file_id IS NOT NULL
+             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id AND p.pcrc32 IS NULL)
+           LIMIT ?)`,
+        [500]
+      )
+      total += result.rowCount ?? 0
+      if (!result.rowCount) break
+    }
+    return total
+  }
+
+  /** Header-derived binaries that never got a second article (gcConfig.binaries.singleArticleAfterSeconds). */
+  private async deleteSingleArticleBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.singleArticleAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const { rows } = await db.rawQuery(
+        `WITH batch AS (
+           SELECT id FROM binaries WHERE id > ? ORDER BY id LIMIT ?
+         ), gone AS (
+           DELETE FROM binaries b USING batch
+           WHERE b.id = batch.id AND b.completed_at IS NULL AND b.size IS NULL AND b.total_parts > 1
+             AND b.last_seen_at < ? AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) <= 1
+           RETURNING 1)
+         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
+        [this.singleCursor, gcConfig.deleteBatch, cutoff]
+      )
+      total += Number(rows[0].deleted)
+      if (rows[0].last === null) {
+        this.singleCursor = 0
+        break
+      }
+      this.singleCursor = Number(rows[0].last)
+    }
+    return total
+  }
+
+  /**
+   * Completed header-derived binaries without a fingerprint, some hours after
+   * completion: their file exists, their parts are gone, and par2 naming needs
+   * a size and MD5 they don't have. Oldest first, via the last_seen_at index.
+   */
+  private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM binaries WHERE id IN (
+           SELECT id FROM binaries
+           WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
+             AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
+           ORDER BY last_seen_at LIMIT ?)`,
+        [cutoff, gcConfig.deleteBatch]
+      )
+      total += result.rowCount
+      if (result.rowCount < gcConfig.deleteBatch) break
+    }
+    return total
+  }
+
+  /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
+  private async deleteEmptyReleases(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const { rows } = await db.rawQuery(
+        `WITH batch AS (
+           SELECT id FROM releases WHERE id > ? ORDER BY id LIMIT ?
+         ), gone AS (
+           DELETE FROM releases r USING batch
+           WHERE r.id = batch.id AND r.file_count = 0 AND r.first_seen_at < ?
+             AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
+             AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)
+             AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)
+             AND NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)
+           RETURNING 1)
+         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
+        [this.releaseCursor, gcConfig.deleteBatch, cutoff]
+      )
+      total += Number(rows[0].deleted)
+      if (rows[0].last === null) {
+        this.releaseCursor = 0
+        break
+      }
+      this.releaseCursor = Number(rows[0].last)
+    }
+    return total
+  }
+
+  /** Removes jobs in `state` that finished more than `ageSeconds` ago. */
+  private async cleanJobs(queue: Queue, state: 'completed' | 'failed', ageSeconds: number) {
+    const graceMs = ageSeconds * 1000
+    if (this.dryRun) {
+      const cutoff = Date.now() - graceMs
+      // Finished-job sets are sorted by finishedOn, so a score range counts them cheaply.
+      return RedisService.client.zcount(queue.toKey(state), '-inf', cutoff)
+    }
+
+    let total = 0
+    let removed: string[]
+    do {
+      removed = await queue.clean(graceMs, CLEAN_BATCH, state)
+      total += removed.length
+    } while (removed.length === CLEAN_BATCH)
+    return total
+  }
+
+  /**
+   * Pending-file hashes of the old header worker (and the pre-AdonisJS app)
+   * never expire. Give them a TTL rather than deleting them outright.
+   */
+  private async expireLegacyPendingFiles() {
+    const redis = RedisService.client
+    const { prefix, ttlSeconds } = gcConfig.legacyPendingFiles
+    let cursor = '0'
+    let scanned = 0
+    let expired = 0
+
+    do {
+      const [next, keys]: [string, string[]] = await redis.scan(
+        cursor,
+        'MATCH',
+        `${prefix}*`,
+        'COUNT',
+        SCAN_COUNT
+      )
+      cursor = next
+      if (keys.length === 0) continue
+      scanned += keys.length
+
+      const ttls: [Error | null, number][] = await keys
+        .reduce((pipeline, key) => pipeline.ttl(key), redis.pipeline())
+        .exec()
+      const orphaned = keys.filter((_, i) => ttls[i][1] === -1)
+      expired += orphaned.length
+
+      if (!this.dryRun && orphaned.length > 0) {
+        await orphaned
+          .reduce((pipeline, key) => pipeline.expire(key, ttlSeconds, 'NX'), redis.pipeline())
+          .exec()
+      }
+    } while (cursor !== '0')
+
+    return { scanned, expired }
+  }
+
+  private verb(present: string, past: string) {
+    return this.dryRun ? `would ${present}` : past
+  }
+}

commands/GroupAdd.ts

@@ -0,0 +1,79 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import Group from '#models/group'
+import NntpService from '#services/NntpService'
+import { parseGroupResponse } from '#lib/nntp/group'
+
+/**
+ * Adds newsgroups to index (or reactivates them). The scheduler picks them
+ * up on its next tick, starting INDEX_INITIAL_BACKFILL articles back.
+ */
+export default class GroupAdd extends BaseCommand {
+  public static commandName = 'group:add'
+  public static description = 'Adds newsgroups to index.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Group names, e.g. alt.binaries.hdtv' })
+  declare names: string[]
+
+  @flags.boolean({ description: 'Deactivate the groups instead' })
+  declare disable: boolean
+
+  @flags.boolean({
+    description: 'Also download bodies to identify obfuscated posts (one body per article; bandwidth-heavy)',
+  })
+  declare bodies: boolean
+
+  @flags.number({ description: 'Start this many articles back instead of INDEX_INITIAL_BACKFILL (new groups only)' })
+  declare backfill?: number
+
+  @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
+  declare reset: boolean
+
+  @flags.boolean({ description: "Start at the server's oldest article: the group's full history (also for groups indexed before)" })
+  declare fromStart: boolean
+
+  public async run() {
+    for (const name of this.names) {
+      if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
+        this.logger.error(`Not a valid group name: ${name}`)
+        continue
+      }
+      const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
+      group.active = !this.disable
+      if (this.bodies) group.fetchBodies = true
+      if (this.fromStart) {
+        // The scheduler continues after last_queued_id; 0 would mean "new group" to it.
+        const { first } = await this.groupInfo(name)
+        group.lastQueuedId = first > 1n ? first - 1n : 1n
+        group.lastIndexedId = group.lastQueuedId
+      } else if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
+        const { last } = await this.groupInfo(name)
+        group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
+        group.lastIndexedId = group.lastQueuedId
+      }
+      await group.save()
+      this.logger.info(
+        `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
+          (this.backfill !== undefined || this.fromStart ? `, starting after article ${group.lastQueuedId}` : '')
+      )
+    }
+    if (this.backfill !== undefined || this.fromStart) await NntpService.shutdown()
+  }
+
+  private async groupInfo(name: string) {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      return parseGroupResponse((await conn.group(name)).resp)
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+}

commands/GroupBackfill.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import BackfillService from '#services/BackfillService'
+import { isSpotGroup } from '#services/SpotService'
+
+/**
+ * Indexes a group's older headers (CONCEPT.md D32): from where live
+ * indexing began back to N days ago, header-only and behind live indexing.
+ * index:scheduler queues the ranges; this only sets where to go.
+ */
+export default class GroupBackfill extends BaseCommand {
+  public static commandName = 'group:backfill'
+  public static description = "Indexes groups' older headers (header-only, behind live indexing)."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Groups (default: every active binary group)', required: false })
+  declare names?: string[]
+
+  @flags.number({ description: 'How many days back', default: 21 })
+  declare days: number
+
+  @flags.boolean({ description: 'Stop backfilling these groups (ranges already queued still run)' })
+  declare stop: boolean
+
+  public async run() {
+    const names = this.names?.length
+      ? this.names
+      : (await db.from('groups').where('active', true).orderBy('name').select('name')).map((g) => g.name as string).filter((n) => !isSpotGroup(n))
+    try {
+      for (const name of names) {
+        if (this.stop) {
+          await BackfillService.stop(name)
+          this.logger.info(`${name}: backfill stopped.`)
+          continue
+        }
+        const range = await BackfillService.start(name, this.days)
+        this.logger.info(
+          range
+            ? `${name}: backfilling articles ${range.from}–${range.to} (${(range.to - range.from).toLocaleString('en')} articles, ${this.days} days).`
+            : `${name}: nothing to backfill.`
+        )
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/GroupsTop.ts

@@ -0,0 +1,176 @@
+import { writeFile } from 'node:fs/promises'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import NntpService from '#services/NntpService'
+import indexerConfig from '#config/indexer'
+import { parseGroupResponse } from '#lib/nntp/group'
+import { parseSubject } from '#lib/subject/index'
+
+interface GroupRate {
+  name: string
+  perMinute: number
+  sampled?: number
+  normal?: number
+  obfuscated?: number
+  skipped?: number
+  avgArticleBytes?: number
+  avgHeaderBytes?: number
+}
+
+/**
+ * Lists the busiest newsgroups (by articles posted per minute, from two
+ * LIST ACTIVE snapshots) and optionally samples their newest headers to
+ * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)).
+ *
+ *   node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt
+ */
+export default class GroupsTop extends BaseCommand {
+  public static commandName = 'groups:top'
+  public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' })
+  declare pattern: string
+
+  @flags.number({ description: 'Minutes between the two snapshots', default: 5 })
+  declare minutes: number
+
+  @flags.number({ description: 'Number of groups to list', default: 1000 })
+  declare limit: number
+
+  @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 })
+  declare sample: number
+
+  @flags.string({ description: 'Write the list to this file (default: print only)' })
+  declare out?: string
+
+  public async run() {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      this.logger.info(`Snapshot 1 of ${this.pattern}…`)
+      const first = await this.snapshot(conn)
+      this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`)
+      const started = Date.now()
+      await sleep(this.minutes * 60_000)
+      const second = await this.snapshot(conn)
+      const minutes = (Date.now() - started) / 60_000
+
+      const groups: GroupRate[] = []
+      for (const [name, high] of second) {
+        const before = first.get(name)
+        if (before === undefined || high <= before) continue
+        groups.push({ name, perMinute: Number(high - before) / minutes })
+      }
+      groups.sort((a, b) => b.perMinute - a.perMinute)
+      const top = groups.slice(0, this.limit)
+      this.logger.info(`${groups.length} active groups; listing ${top.length}.`)
+
+      if (this.sample > 0) {
+        for (const [i, group] of top.entries()) {
+          try {
+            Object.assign(group, await this.sampleGroup(conn, group.name))
+          } catch (error: any) {
+            this.logger.warning(`${group.name}: ${error.message}`)
+          }
+          if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`)
+        }
+      }
+
+      const text = this.format(top, minutes)
+      if (this.out) {
+        await writeFile(this.out, text)
+        this.logger.info(`Written to ${this.out}`)
+      } else {
+        console.log(text)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+      await NntpService.shutdown()
+    }
+  }
+
+  /** Group name → highest article number, from LIST ACTIVE. */
+  private async snapshot(conn: any): Promise<Map<string, bigint>> {
+    const { data } = await conn.list(this.pattern)
+    const result = new Map<string, bigint>()
+    for (const line of Buffer.from(data).toString('latin1').split('\r\n')) {
+      const [name, high] = line.split(' ')
+      if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high))
+    }
+    return result
+  }
+
+  /** Classifies the newest headers the way worker:fetch does. */
+  private async sampleGroup(conn: any, name: string) {
+    const { last } = parseGroupResponse((await conn.group(name)).resp)
+    const start = last - BigInt(this.sample) + 1n
+    const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString())
+    let normal = 0
+    let skipped = 0
+    let obfuscated = 0
+    let obfuscatedBytes = 0
+    let headerBytes = 0
+    for (const [, header] of overviews as [number, Record<string, string>][]) {
+      if (!header) continue
+      headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0)
+      if (header.subject && parseSubject(header.subject)) normal++
+      else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
+      else {
+        obfuscated++
+        obfuscatedBytes += Number(header[':bytes']) || 0
+      }
+    }
+    const sampled = normal + skipped + obfuscated
+    return {
+      sampled,
+      normal,
+      skipped,
+      obfuscated,
+      avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0,
+      avgHeaderBytes: sampled ? headerBytes / sampled : 0,
+    }
+  }
+
+  private format(groups: GroupRate[], minutes: number): string {
+    const sampled = groups.some((g) => g.sampled)
+    const lines = [
+      `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`,
+      sampled
+        ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s'
+        : '# rank\tgroup\tarticles/min',
+    ]
+    let bodyTotal = 0
+    let headerTotal = 0
+    let articlesTotal = 0
+    for (const [i, g] of groups.entries()) {
+      articlesTotal += g.perMinute
+      if (!g.sampled) {
+        lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`)
+        continue
+      }
+      const obfuscatedShare = g.obfuscated! / g.sampled
+      // Bodies: every obfuscated article is downloaded once. Headers: every article.
+      const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6
+      const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6
+      bodyTotal += bodyMbit
+      headerTotal += headerMbit
+      lines.push(
+        [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t')
+      )
+    }
+    lines.push(`# total: ${Math.round(articlesTotal)} articles/min`)
+    if (sampled) {
+      lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`)
+      lines.push(`#   of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`)
+    }
+    return lines.join('\n') + '\n'
+  }
+}

commands/IndexScheduler.ts

@@ -0,0 +1,130 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import Group from '#models/group'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
+import Shutdown from '#services/Shutdown'
+import FetchRangeService from '#services/FetchRangeService'
+import SplitService from '#services/SplitService'
+import BackfillService from '#services/BackfillService'
+
+export default class IndexScheduler extends BaseCommand {
+  public static commandName = 'index:scheduler'
+  public static description = 'Periodically checks for new articles and schedules them for fetching.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  private pool = NntpService
+
+  public async run() {
+    this.logger.info('Scheduler started. Awaiting tasks...')
+
+    const schedule = async () => {
+      this.logger.info('Checking for new headers...')
+      const groups = await Group.query().where('active', true)
+
+      if (groups.length === 0) {
+        this.logger.info('No active groups to index. Add some via `node ace db:seed` or manually.')
+        return
+      }
+
+      let conn
+      let failed = false
+      try {
+        conn = await this.pool.acquire()
+
+        for (const group of groups) {
+          try {
+            const { first: firstArticle, last: lastArticle } = await this.pool.selectGroup(
+              conn,
+              group.name
+            )
+
+            await FetchRangeService.requeue(group)
+
+            // Postgres returns bigint columns as strings.
+            const lastQueued = BigInt(group.lastQueuedId ?? 0)
+            const { batchSize, maxBatchesPerTick, initialBackfill } = indexerConfig
+            let startId =
+              lastQueued === 0n ? lastArticle - initialBackfill + 1n : lastQueued + 1n
+            if (startId < firstArticle) startId = firstArticle
+
+            if (startId > lastArticle) {
+              await FetchRangeService.advanceWatermark(group)
+              this.logger.info(`No new headers for group ${group.name}.`)
+              continue
+            }
+
+            let batches = 0
+            let endId = startId - 1n
+            while (endId < lastArticle && batches < maxBatchesPerTick) {
+              const batchStart = endId + 1n
+              endId = batchStart + batchSize - 1n < lastArticle ? batchStart + batchSize - 1n : lastArticle
+              await FetchRangeService.queue(group, batchStart, endId)
+              batches++
+            }
+
+            this.logger.info(
+              `Queued ${batches} fetch job(s) for ${group.name}: articles ${startId} to ${endId}` +
+                (endId < lastArticle ? ` (${lastArticle - endId} left for next tick)` : '')
+            )
+
+            group.lastQueuedId = endId
+            await group.save()
+            const watermark = await FetchRangeService.advanceWatermark(group)
+            if (endId - watermark > batchSize * BigInt(maxBatchesPerTick) * 3n) {
+              this.logger.warning(
+                `${group.name}: indexed up to ${watermark}, ${endId - watermark} articles behind the queue.`
+              )
+            }
+          } catch (err: any) {
+            failed = true
+            this.logger.error(`Error processing group ${group.name}: ${err.message}`)
+          }
+        }
+      } catch (err: any) {
+        failed = true
+        this.logger.error(`Error in scheduler main loop: ${err.message}`)
+      } finally {
+        if (conn) {
+          this.pool.release(conn, failed)
+        }
+      }
+    }
+
+    const split = async () => {
+      try {
+        const result = await SplitService.splitPending()
+        if (result.parents > 0) {
+          this.logger.info(
+            `Split ${result.parents} conflicted binaries: ${result.volumes} verified volumes, ${result.completed} completed.`
+          )
+        }
+      } catch (err: any) {
+        this.logger.error(`Error splitting conflicted binaries: ${err.message}`)
+      }
+    }
+
+    const backfill = async () => {
+      try {
+        const queued = await BackfillService.tick()
+        if (queued > 0) this.logger.info(`Queued ${queued} backfill range(s).`)
+      } catch (err: any) {
+        this.logger.error(`Error queuing backfill: ${err.message}`)
+      }
+    }
+
+    // Run immediately, then wait a full interval after each run so runs never overlap.
+    while (!Shutdown.requested) {
+      await schedule()
+      await backfill()
+      await split()
+      await Shutdown.sleep(indexerConfig.intervalMs)
+    }
+    await this.pool.shutdown()
+    await QueueService.closeAll()
+  }
+}

commands/ReleaseMergeSplit.ts

@@ -0,0 +1,21 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+
+/** Merges uploads split per poster (gc:scheduler also does this every sweep). */
+export default class ReleaseMergeSplit extends BaseCommand {
+  public static commandName = 'release:merge-split'
+  public static description = 'Merges releases of one upload that was posted under a different poster per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const { names, releases } = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    this.logger.info(`${names} split uploads; ${releases} releases ${this.dryRun ? 'would be' : ''} merged into their first.`)
+  }
+}

commands/ReleaseRebuild.ts

@@ -0,0 +1,163 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import IngestService from '#services/IngestService'
+import NameService from '#services/NameService'
+import QueueService from '#services/QueueService'
+import { rarReleaseIdentity, releaseIdentity } from '#lib/release/index'
+
+const BATCH = 5000
+
+/**
+ * Recomputes the release of every binary (and its file) with the current
+ * rules in lib/release. Used after changing those rules.
+ */
+export default class ReleaseRebuild extends BaseCommand {
+  public static commandName = 'release:rebuild'
+  public static description = 'Reassigns all binaries and files to releases using the current grouping rules.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({
+    description:
+      'Also drop everything derived from file contents (par2 lists, par2/RAR groupings, real names), ' +
+      'to rebuild it with content:backfill --all',
+  })
+  declare resetContent: boolean
+
+  public async run() {
+    if (this.resetContent) await this.dropContentDerived()
+
+    let lastId = 0
+    let assigned = 0
+    let unassigned = 0
+    try {
+      while (true) {
+        const binaries = await db
+          .from('binaries')
+          .where('id', '>', lastId)
+          .orderBy('id')
+          .limit(BATCH)
+          .select('id', 'poster', 'subject', 'name', 'groups', 'posted_at')
+        if (binaries.length === 0) break
+        lastId = Number(binaries.at(-1).id)
+
+        const segments = binaries.map((binary) => {
+          // Body-derived binaries store a synthetic subject, see segmentFromYenc.
+          const weak = /^"[^"]*" yEnc$/.test(binary.subject)
+          return {
+            id: Number(binary.id),
+            poster: binary.poster,
+            groups: binary.groups,
+            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
+            weakIdentity: weak,
+            release: releaseIdentity({ poster: binary.poster, subject: binary.subject, name: binary.name, weak }),
+          }
+        })
+
+        await db.transaction(async (trx) => {
+          const releaseIds = await IngestService.upsertReleases(trx.knexClient, segments)
+          const ids = segments.map((s) => s.id)
+          const releaseIdsPerBinary = segments.map((s) =>
+            s.release ? releaseIds.get(s.release.key)! : null
+          )
+          await trx.rawQuery(
+            `UPDATE binaries b SET release_id = v.release_id
+             FROM unnest(?::bigint[], ?::bigint[]) AS v(id, release_id)
+             WHERE b.id = v.id AND b.release_id IS DISTINCT FROM v.release_id
+               AND NOT (v.release_id IS NULL
+                        AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))
+               -- A posted NZB decided this grouping (NzbImportService); name rules don't override it.
+               AND NOT EXISTS (SELECT 1 FROM files n WHERE n.release_id = b.release_id AND n.content_type = 'nzb')`,
+            [ids, releaseIdsPerBinary] as any
+          )
+        })
+        assigned += segments.filter((s) => s.release).length
+        unassigned += segments.filter((s) => !s.release).length
+      }
+
+      // Files grouped by their RAR headers: regroup from the stored listing (no downloads).
+      let rarGrouped = 0
+      let rarCursor = 0
+      while (true) {
+        const rows = await db
+          .from('files as f')
+          .join('binaries as b', 'b.file_id', 'f.id')
+          .whereNotNull('f.archive')
+          .whereNull('b.release_id')
+          .where('f.id', '>', rarCursor)
+          .orderBy('f.id')
+          .limit(BATCH)
+          .select('f.id as file_id', 'f.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
+        if (rows.length === 0) break
+        rarCursor = Number(rows.at(-1).file_id)
+        const segments = rows.map((r) => ({
+          fileId: Number(r.file_id),
+          binaryId: Number(r.binary_id),
+          poster: r.poster,
+          groups: r.groups,
+          postedAt: r.posted_at ? new Date(r.posted_at) : undefined,
+          weakIdentity: true,
+          release: rarReleaseIdentity(r.archive?.entries ?? []),
+        }))
+        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), segments)
+        for (const s of segments.filter((x) => x.release)) {
+          const releaseId = ids.get(s.release!.key)
+          await db.from('binaries').where('id', s.binaryId).update({ release_id: releaseId })
+          await db.from('files').where('id', s.fileId).update({ release_id: releaseId })
+          rarGrouped++
+        }
+      }
+
+      // Real names from par2 for binaries with a known size (no downloads).
+      let named = 0
+      let nameCursor = 0
+      while (true) {
+        const rows = await db.from('binaries').whereNotNull('size').where('id', '>', nameCursor).orderBy('id').limit(BATCH).select('id')
+        if (rows.length === 0) break
+        nameCursor = Number(rows.at(-1).id)
+        named += await NameService.resolve(rows.map((r) => Number(r.id)))
+      }
+
+      const files = await db.rawQuery(
+        `UPDATE files f SET release_id = b.release_id FROM binaries b
+         WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`
+      )
+      const orphans = await db.rawQuery(
+        `DELETE FROM releases r WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
+           AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)`
+      )
+      if (this.resetContent) {
+        this.logger.info('Content-derived data dropped; run `node ace content:backfill --all` to rebuild it.')
+      }
+      this.logger.info(
+        `Binaries: ${assigned} in a release, ${unassigned} without, ${rarGrouped} regrouped from RAR headers, ` +
+          `${named} named from par2. ` +
+          `Files updated: ${files.rowCount}. Empty releases removed: ${orphans.rowCount}.`
+      )
+    } finally {
+      await QueueService.closeAll()
+    }
+  }
+
+  /** par2 lists, par2/RAR release assignments and recovered names. */
+  private async dropContentDerived() {
+    await db.transaction(async (trx) => {
+      await trx.rawQuery('DELETE FROM release_files')
+      await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`)
+      await trx.rawQuery(`UPDATE files SET real_name = NULL, archive = NULL WHERE real_name IS NOT NULL OR archive IS NOT NULL`)
+      await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`)
+      // Binaries whose name rules find no release were grouped from contents (or wrongly by size).
+      await trx.rawQuery(`
+        UPDATE binaries SET release_id = NULL
+        WHERE subject ~ '^"[^"]*" yEnc$' AND release_id IS NOT NULL
+          AND (name !~ '\\.[A-Za-z0-9]{1,5}$' OR release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))`)
+      await trx.rawQuery(`
+        UPDATE files f SET release_id = b.release_id FROM binaries b
+        WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`)
+    })
+    this.logger.info('Dropped par2 lists, content-derived groupings and recovered names.')
+  }
+}

commands/ReleaseRename.ts

@@ -0,0 +1,20 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import { renameFromStoredContents } from '#services/ArchiveService'
+import { regroupAndRenameFromPar2 } from '#services/NameService'
+
+/** Renames random-named releases from their stored archive listings (no downloads). */
+export default class ReleaseRename extends BaseCommand {
+  public static commandName = 'release:rename'
+  public static description = 'Renames random-named releases from their par2 listings and archive contents.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const par2 = await regroupAndRenameFromPar2()
+    this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`)
+    this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`)
+  }
+}

commands/ReleaseStats.ts

@@ -0,0 +1,57 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+
+/** Prints how well files are being grouped into releases. */
+export default class ReleaseStats extends BaseCommand {
+  public static commandName = 'release:stats'
+  public static description = 'Shows release recognition statistics.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const { rows } = await db.rawQuery(`
+      WITH r AS (
+        SELECT r.id, r.file_total, r.poster IS NULL AS weak, r.source,
+               count(b.id) AS binaries,
+               count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed
+        FROM releases r LEFT JOIN binaries b ON b.release_id = r.id
+        GROUP BY r.id
+      )
+      SELECT
+        count(*) AS releases,
+        count(*) FILTER (WHERE completed > 0) AS with_files,
+        count(*) FILTER (WHERE completed >= 2) AS multi_file,
+        count(*) FILTER (WHERE file_total IS NOT NULL AND completed >= file_total) AS complete_by_counter,
+        count(*) FILTER (WHERE weak) AS from_obfuscated,
+        count(*) FILTER (WHERE weak AND completed > 0) AS from_obfuscated_with_files
+      FROM r`)
+    const { rows: coverage } = await db.rawQuery(`
+      SELECT count(*) AS completed_files,
+             count(*) FILTER (WHERE release_id IS NOT NULL) AS in_release
+      FROM binaries WHERE completed_at IS NOT NULL`)
+    const { rows: bySource } = await db.rawQuery(`
+      SELECT coalesce(r.source, 'unknown') AS source, count(DISTINCT r.id) AS releases,
+             count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed_files
+      FROM releases r JOIN binaries b ON b.release_id = r.id GROUP BY 1 ORDER BY 2 DESC`)
+    // Readable names: descriptive as posted, or recovered from contents.
+    const { rows: naming } = await db.rawQuery(`
+      WITH f AS (
+        SELECT real_name IS NOT NULL AS recovered,
+               is_readable_name(coalesce(real_name, filename)) AS readable_name,
+               regexp_replace(coalesce(real_name, filename),
+                 '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|mkv|mp4|avi|flac|mp3|jpg))+$', '', 'i') AS stem,
+               coalesce(real_name, filename) AS name
+        FROM files
+      )
+      SELECT count(*) AS files,
+             count(*) FILTER (WHERE recovered) AS names_recovered,
+             count(*) FILTER (WHERE name ~ '\\.[A-Za-z0-9]{1,5}$' AND readable_name) AS readable
+      FROM f`)
+    this.logger.info(JSON.stringify({ ...rows[0], ...coverage[0] }))
+    this.logger.info(`naming: ${JSON.stringify(naming[0])}`)
+    this.logger.info(`by source: ${JSON.stringify(bySource)}`)
+  }
+}

commands/SpotPasswords.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import SpotService from '#services/SpotService'
+
+/** Passwords from the titles and descriptions of spots read so far, onto their releases. */
+export default class SpotPasswords extends BaseCommand {
+  public static commandName = 'spot:passwords'
+  public static description = "Takes archive passwords from spots' titles and descriptions onto their releases."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const spots = await db
+      .from('spots')
+      .whereNotNull('release_id')
+      .where((q) => q.whereNotNull('description').orWhere('title', 'like', '%{{%}}%'))
+      .select('message_id')
+    let found = 0
+    for (const spot of spots) if (await SpotService.applyPassword(spot.message_id)) found++
+    this.logger.info(`${found} of ${spots.length} spots name a password in their title or description.`)
+  }
+}

commands/SpotReimport.ts

@@ -0,0 +1,51 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+/**
+ * Imported spots whose files sit in a release that isn't the spot's own
+ * (another source, or another name): their NZB is read again, which moves
+ * the files into the spot's release under its title (CONCEPT.md D29).
+ */
+export default class SpotReimport extends BaseCommand {
+  public static commandName = 'spot:reimport'
+  public static description = "Moves imported spots' files into their own release, named after the spot."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const spots = await db
+      .from('spots as s')
+      .join('releases as r', 'r.id', 's.release_id')
+      .where('s.status', 'imported')
+      .where((q) => q.whereNot('r.source', 'spot').orWhereRaw('r.name <> s.title'))
+      .orderBy('s.posted_at', 'desc')
+      .select('s.*', 'r.name as release_name')
+    this.logger.info(`${spots.length} imported spots are in a release that isn't their own.`)
+    if (this.dryRun) return
+    let moved = 0
+    let failed = 0
+    try {
+      for (const spot of spots) {
+        try {
+          const releaseId = await SpotService.reimport(spot)
+          if (releaseId) moved++
+          else failed++
+        } catch (error: any) {
+          failed++
+          this.logger.warning(`${spot.title}: ${error.message}`)
+        }
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+    this.logger.info(`${moved} spots re-imported, ${failed} without a usable NZB (dummy, gone) or failed.`)
+  }
+}

commands/workers/Body.ts

@@ -0,0 +1,97 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import { Worker } from 'bullmq'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
+import env from '#start/env'
+import NntpService from '#services/NntpService'
+import IngestService, { segmentFromYenc } from '#services/IngestService'
+import { parseYencMeta, YencError } from '#lib/yenc/index'
+import { firstChunkMd5 } from '#services/ContentService'
+
+export default class BodyWorker extends BaseCommand {
+  public static commandName = 'worker:body'
+  public static description = 'Starts a worker to process article bodies for yEnc metadata.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    this.logger.info('Starting body worker...')
+
+    const pool = NntpService
+
+    const worker = new Worker('body-queue', async (job) => {
+      const { header, group, articleNumber } = job.data
+      const messageId = header['message-id']
+      this.logger.debug(`Processing header with unparsable subject: ${header.subject}`)
+
+      let conn
+      let failed = false
+      try {
+        conn = await pool.acquire()
+        const bodyBuffer = await pool.fetchBody(conn, messageId)
+
+        let meta
+        try {
+          meta = parseYencMeta(bodyBuffer)
+        } catch (parseError: any) {
+          if (parseError instanceof YencError && bodyBuffer.indexOf('=ybegin ') === -1) {
+            this.logger.debug(`Not a yEnc post, skipping: ${messageId}`)
+            return
+          }
+          this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`)
+          const debugDir = path.join(this.app.appRoot.pathname, 'debug')
+          await fs.mkdir(debugDir, { recursive: true })
+          const timestamp = new Date().toISOString().replace(/:/g, '-')
+          const dumpFile = path.join(debugDir, `body-error-${timestamp}-${messageId.replace(/[<>]/g, '')}.bin`)
+          await fs.writeFile(dumpFile, bodyBuffer)
+          this.logger.error(`Problematic body buffer saved to: ${dumpFile}`)
+          throw parseError
+        }
+
+        pool.release(conn)
+        conn = undefined
+
+        const segment = segmentFromYenc(header, group, meta, articleNumber)
+        if (segment?.number === 1) segment.md5_16k = firstChunkMd5(bodyBuffer, meta.begin.size)
+        if (!segment) {
+          this.logger.warning(`Inconsistent yEnc part numbering in ${messageId}, skipping.`)
+          return
+        }
+        const completed = await IngestService.ingest([segment])
+        this.logger.debug(
+          `Stored part ${segment.number}/${segment.totalParts} of "${segment.name}"` +
+            (completed.length > 0 ? ' (file complete)' : '')
+        )
+      } catch (error: any) {
+        failed = true
+        this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`)
+        throw error
+      } finally {
+        if (conn) {
+          pool.release(conn, failed)
+        }
+      }
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading, so keep this <= the pool size.
+      concurrency: env.get('BODY_CONCURRENCY', 1),
+    })
+
+    worker.on('failed', (job, err) => {
+      this.logger.error(`Body job ${job?.id} failed: ${err.message}`)
+    })
+
+    this.logger.info('Body worker started and listening for jobs.')
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
+  }
+}

commands/workers/Collection.ts

@@ -0,0 +1,80 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import { Worker } from 'bullmq'
+import queueConfig from '#config/queue'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
+import NntpService from '#services/NntpService'
+import ContentService from '#services/ContentService'
+import File from '#models/file'
+
+export default class CollectionWorker extends BaseCommand {
+  public static commandName = 'worker:collection'
+  public static description = 'Starts a worker to process file collections (e.g., RAR archives).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    this.logger.info('Starting collection worker...')
+
+    const worker = new Worker('collection-queue', async (job) => {
+      const { fileId } = job.data
+      this.logger.debug(`Processing file ID ${fileId} for collection.`)
+
+      const file = await File.find(fileId)
+      if (!file) {
+        this.logger.error(`File with ID ${fileId} not found in the database.`)
+        return
+      }
+
+      try {
+        const content = await ContentService.inspect(fileId)
+        if (content?.par2) {
+          const { par2 } = content
+          this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
+        } else if (content?.archive) {
+          const { info, releaseRenamed, grouped } = content.archive
+          const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
+            .filter(Boolean)
+            .join(', ')
+          this.logger.info(
+            `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
+              (grouped ? ', grouped into a release' : '') +
+              (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
+          )
+        }
+        if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
+        if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
+        if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
+        if (content?.yencName) this.logger.info(`"${file.filename}" is "${content.yencName}" (yEnc name).`)
+        if (content?.nicovideo) this.logger.info(`"${file.filename}" → "${content.nicovideo}".`)
+        if (content?.sfvMoved) this.logger.info(`SFV "${file.filename}" moved ${content.sfvMoved} files into its release.`)
+        if (content?.contentType === 'yencbin') this.logger.info(`"${file.filename}" is a yEncBin encrypted container.`)
+        if (content?.nzb) {
+          const n = content.nzb
+          this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
+        }
+      } catch (error: any) {
+        this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
+      }
+    }, {
+      connection: queueConfig.connection,
+      // Each job holds one NNTP connection while downloading; keep <= the pool size.
+      concurrency: env.get('COLLECTION_CONCURRENCY', 1),
+    })
+
+    worker.on('failed', (job, err) => {
+      this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
+    })
+
+    this.logger.info('Collection worker started and listening for jobs.')
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
+  }
+}

commands/workers/Deferred.ts

@@ -0,0 +1,57 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import QueueService from '#services/QueueService'
+import DeferredService from '#services/DeferredService'
+
+const IDLE_MS = 60_000
+
+/**
+ * Works through `deferred_files` (CONCEPT.md D28): one body per file, and
+ * small files completely. Not started by default (compose profile
+ * `deferred`); run it when there is bandwidth to spare.
+ */
+export default class DeferredWorker extends BaseCommand {
+  public static commandName = 'worker:deferred'
+  public static description = 'Samples deferred files (e.g. ngPost uploads): one article body per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Stop after this many files (default: keep running)' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = env.get('DEFERRED_CONCURRENCY', 4)
+    this.logger.info(`Deferred worker started (concurrency ${concurrency}).`)
+    const counts: Record<string, number> = {}
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        if (batch.length === 0) {
+          if (this.limit) break
+          await Shutdown.sleep(IDLE_MS)
+          continue
+        }
+        let next = 0
+        await Promise.all(
+          Array.from({ length: concurrency }, async () => {
+            while (next < batch.length) {
+              const status = await DeferredService.process(batch[next++])
+              counts[status] = (counts[status] ?? 0) + 1
+            }
+          })
+        )
+        done += batch.length
+        this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+      }
+    } finally {
+      await NntpService.shutdown()
+      await QueueService.closeAll()
+    }
+  }
+}

commands/workers/Fetch.ts

@@ -0,0 +1,118 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import { Worker } from 'bullmq'
+import queueConfig from '#config/queue'
+import env from '#start/env'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
+import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
+import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
+import SpotService, { isSpotGroup } from '#services/SpotService'
+import StagingService from '#services/StagingService'
+
+export default class FetchWorker extends BaseCommand {
+  public static commandName = 'worker:fetch'
+  public static description = 'Starts a worker that fetches headers (XOVER) and stores them as binaries/parts.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    this.logger.info('Starting fetch worker...')
+
+    const pool = NntpService
+    const bodyQueue = QueueService.bodyQueue
+
+    const worker = new Worker('nntp-fetch-queue', async (job) => {
+      const { groupName, startId, endId } = job.data
+      this.logger.info(`Processing fetch job for ${groupName}, articles ${startId}-${endId}`)
+
+      let overviews: [number, Record<string, string>][]
+      let conn
+      let failed = false
+      try {
+        conn = await pool.acquire()
+        await pool.timed(conn.group(groupName), `GROUP ${groupName}`)
+        const reply = await pool.timed<any>(conn.xover(startId, endId), `XOVER ${groupName} ${startId}-${endId}`)
+        // An empty reply parses as one line without fields ([null, {}]).
+        overviews = reply.overviews.filter(([number, header]: [number | null, Record<string, string>]) => number !== null && header?.['message-id'])
+      } catch (error: any) {
+        failed = true
+        this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
+        throw error
+      } finally {
+        if (conn) {
+          pool.release(conn, failed)
+        }
+      }
+
+      if (isSpotGroup(groupName)) {
+        const spots = await SpotService.recordHeaders(groupName, overviews)
+        if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
+        this.logger.info(`${groupName} ${startId}-${endId}: ${overviews.length} headers, ${spots} spots queued.`)
+        return
+      }
+
+      const group = await db.from('groups').where('name', groupName).first()
+      // Backfill is header-only: old bodies would be far more than the link carries (CONCEPT.md D32).
+      const fetchBodies = Boolean(group?.fetch_bodies) && !job.data.backfill
+
+      const segments: IngestSegment[] = []
+      const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
+      const deferred: DeferredArticle[] = []
+      let skipped = 0
+      for (const [articleNumber, header] of overviews) {
+        if (!header) continue
+        const segment = segmentFromHeader(header, groupName, articleNumber)
+        if (segment) segments.push(segment)
+        // Recorded per file (or as article ranges) in every group, bodies or not; processed later if at all.
+        else if (isDeferred(header)) deferred.push({ articleNumber, header })
+        else if (!fetchBodies || indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
+        else obfuscated.push({ articleNumber, header })
+      }
+
+      // Lone segments of new files wait in staging until their file proves real (D35).
+      const { ingest, staged } = StagingService.enabled(groupName)
+        ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(groupName) })
+        : { ingest: segments, staged: 0 }
+      const completed = await IngestService.ingest(ingest)
+      const deferredFiles = await DeferredService.record(groupName, deferred)
+
+      if (obfuscated.length > 0) {
+        await bodyQueue.addBulk(obfuscated.map(({ articleNumber, header }) => bodyJob(header, groupName, articleNumber)))
+      }
+
+      if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
+
+      this.logger.info(
+        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${ingest.length} stored${staged ? `, ${staged} staged` : ''}, ` +
+          `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
+          `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
+      )
+    }, {
+      connection: queueConfig.connection,
+      // XOVER is quick; most of a job is storing. Each concurrent job uses one DB connection (DB_POOL_MAX).
+      concurrency: env.get('FETCH_CONCURRENCY', 5),
+    })
+
+    worker.on('failed', async (job, err) => {
+      this.logger.error(`Fetch job ${job?.id} failed for group ${job?.data.groupName}: ${err.message}`)
+      // Only after the last attempt; the scheduler re-queues the range later.
+      if (job?.data.rangeId && job.attemptsMade >= (job.opts.attempts ?? 1)) {
+        await FetchRangeService.markFailed(job.data.rangeId, err.message).catch(() => {})
+      }
+    })
+
+    this.logger.info('Fetch worker started and listening for jobs.')
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await NntpService.shutdown()
+    await QueueService.closeAll()
+  }
+}

commands/workers/File.ts

@@ -0,0 +1,132 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import { Worker } from 'bullmq'
+import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import queueConfig from '#config/queue'
+import Shutdown from '#services/Shutdown'
+import QueueService from '#services/QueueService'
+
+export default class FileWorker extends BaseCommand {
+  public static commandName = 'worker:file'
+  public static description = 'Starts a worker that turns completed binaries into files.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    this.logger.info('Starting file worker...')
+
+    const collectionQueue = QueueService.collectionQueue
+
+    const worker = new Worker('file-queue', async (job) => {
+      const { binaryId } = job.data
+
+      const binary = await db.from('binaries').where('id', binaryId).first()
+      if (!binary) {
+        this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`)
+        return
+      }
+
+      if (binary.conflicted_at) {
+        this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`)
+        return
+      }
+
+      const parts = await db
+        .from('parts')
+        .where('binary_id', binaryId)
+        .orderBy('number')
+        .select('number', 'message_id', 'bytes')
+      if (parts.length < binary.total_parts) {
+        throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
+      }
+
+      const segments = parts.map((part) => ({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) }))
+
+      // Already known as a file, e.g. added from a posted NZB before its own articles completed.
+      const existing = await db.from('files').where('first_message_id', segments[0]?.id ?? '').first()
+      if (existing) {
+        await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
+        await dropCopiedParts(binary)
+        this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`)
+        return
+      }
+
+      // file_key is unique: a second completion of the same upload inserts nothing.
+      const now = new Date()
+      const [inserted] = await FileStore.insert([
+        {
+          file_key: binary.file_key,
+          filename: binary.name,
+          poster: binary.poster,
+          date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
+          parts: parts.length,
+          segments,
+          groups: JSON.stringify(binary.groups),
+          release_id: binary.release_id,
+          real_name: binary.real_name,
+          size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
+          created_at: now,
+          updated_at: now,
+        },
+      ])
+
+      const fileId =
+        inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
+      await db.from('binaries').where('id', binaryId).update({ file_id: fileId })
+      await dropCopiedParts(binary)
+
+      if (!inserted) {
+        this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`)
+        return
+      }
+
+      this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
+      // An SFV posted earlier may already list it (D34).
+      await SfvService.claim({
+        id: Number(fileId),
+        name: binary.real_name ?? binary.name,
+        poster: binary.poster,
+        date: binary.posted_at ? new Date(binary.posted_at).getTime() : null,
+        release_id: binary.release_id,
+      })
+      await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
+    }, {
+      connection: queueConfig.connection,
+      // One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and
+      // the file insert ignores duplicates, so they can run side by side.
+      concurrency: env.get('FILE_CONCURRENCY', 8),
+    })
+
+    worker.on('failed', (job, err) => {
+      this.logger.error(`File job ${job?.id} failed: ${err.message}`)
+    })
+
+    this.logger.info('File worker started and listening for jobs.')
+    await Shutdown.wait()
+    this.logger.info('Shutting down: finishing active jobs...')
+    await worker.close()
+    await QueueService.closeAll()
+  }
+}
+
+/**
+ * The file holds every message id now. Binaries identified by their subject
+ * (no size: header-derived) don't need their parts any more; body-derived
+ * ones keep them, so articles of an equal-sized next volume are recognised
+ * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
+ */
+/**
+ * The file holds the segment list now. Header-derived parts (no per-part
+ * CRC) aren't needed any more; body-derived ones are kept for split-volume
+ * verification (D20). Going by the CRC, not the binary's size: content
+ * inspection gives header-derived binaries a size too, which kept ~20M
+ * parts around (LOG.md 2026-10-02).
+ */
+async function dropCopiedParts(binary: { id: number }) {
+  await db.from('parts').where('binary_id', binary.id).whereNull('pcrc32').delete()
+}

commands/workers/Spots.ts

@@ -0,0 +1,63 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import env from '#start/env'
+import Shutdown from '#services/Shutdown'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+const IDLE_MS = 30_000
+/** Spots claimed at a time; most of a history backfill is listed in bulk. */
+const BATCH = 500
+/** Imported spots whose details are read per idle round. */
+const DETAILS_BATCH = 50
+
+/**
+ * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
+ * downloads its compressed NZB (a few KB to a few MB) and imports it.
+ */
+export default class SpotsWorker extends BaseCommand {
+  public static commandName = 'worker:spots'
+  public static description = 'Reads Spotnet spots: verifies them and imports their NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.number({ description: 'Stop after this many spots (default: keep running)' })
+  declare limit?: number
+
+  public async run() {
+    const concurrency = env.get('SPOT_CONCURRENCY', 4)
+    this.logger.info(`Spot worker started (concurrency ${concurrency}).`)
+    const counts: Record<string, number> = {}
+    let done = 0
+    try {
+      while (!Shutdown.requested && (!this.limit || done < this.limit)) {
+        const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
+        if (batch.length === 0) {
+          if (this.limit) break
+          // Idle: read the details of imported spots that don't have them yet (newest first).
+          if ((await SpotService.fillDetails(DETAILS_BATCH)) === 0) await Shutdown.sleep(IDLE_MS)
+          continue
+        }
+        // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
+        const listable = batch.filter((spot) => SpotService.isListable(spot))
+        const single = batch.filter((spot) => !SpotService.isListable(spot))
+        counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
+        let next = 0
+        await Promise.all(
+          Array.from({ length: concurrency }, async () => {
+            while (next < single.length) {
+              const status = await SpotService.process(single[next++])
+              counts[status] = (counts[status] ?? 0) + 1
+            }
+          })
+        )
+        done += batch.length
+        this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

config/app.ts

@@ -0,0 +1,84 @@
+import env from '#start/env'
+import app from '@adonisjs/core/services/app'
+import { defineConfig } from '@adonisjs/core/http'
+
+/**
+ * The app URL can be used in various places where you want to create absolute
+ * URLs to your application. For example, when sending emails, images should
+ * use absolute URLs.
+ */
+export const appUrl = env.get('APP_URL')
+
+/**
+ * The configuration settings used by the HTTP server
+ */
+export const http = defineConfig({
+  /**
+   * Generate a unique request id for each incoming request.
+   * Useful to correlate logs and debug a request flow.
+   */
+  generateRequestId: true,
+
+  /**
+   * Allow HTTP method spoofing via the "_method" form/query parameter.
+   * This lets HTML forms target PUT/PATCH/DELETE routes while still
+   * submitting with POST.
+   */
+  allowMethodSpoofing: false,
+
+  /**
+   * Enabling async local storage will let you access HTTP context
+   * from anywhere inside your application.
+   */
+  useAsyncLocalStorage: false,
+
+  /**
+   * Redirect configuration controls the behavior of
+   * response.redirect().back() and query string forwarding.
+   */
+  redirect: {
+    /**
+     * When enabled, all redirects automatically carry over the current
+     * request's query string parameters to the redirect destination.
+     * Use withQs(false) to opt out for a specific redirect.
+     */
+    forwardQueryString: true,
+  },
+
+  /**
+   * Manage cookies configuration. The settings for the session id cookie are
+   * defined inside the "config/session.ts" file.
+   */
+  cookie: {
+    /**
+     * Restrict the cookie to a specific domain.
+     * Keep empty to use the current host.
+     */
+    domain: '',
+
+    /**
+     * Restrict the cookie to a URL path. '/' means all routes.
+     */
+    path: '/',
+
+    /**
+     * Default lifetime for cookies managed by the HTTP layer.
+     */
+    maxAge: '2h',
+
+    /**
+     * Prevent JavaScript access to the cookie in the browser.
+     */
+    httpOnly: true,
+
+    /**
+     * Send cookies only over HTTPS in production.
+     */
+    secure: app.inProduction,
+
+    /**
+     * Cross-site policy for cookie sending.
+     */
+    sameSite: 'lax',
+  },
+})

config/auth.ts

@@ -0,0 +1,50 @@
+import { defineConfig } from '@adonisjs/auth'
+import { sessionGuard, sessionUserProvider } from '@adonisjs/auth/session'
+import { tokensGuard, tokensUserProvider } from '@adonisjs/auth/access_tokens'
+import type { InferAuthenticators, InferAuthEvents, Authenticators } from '@adonisjs/auth/types'
+
+const authConfig = defineConfig({
+  /**
+   * Default guard used when no guard is explicitly specified.
+   */
+  default: 'api',
+
+  guards: {
+    /**
+     * Token-based guard for stateless API authentication.
+     */
+    api: tokensGuard({
+      provider: tokensUserProvider({
+        tokens: 'accessTokens',
+        model: () => import('#models/user'),
+      }),
+    }),
+
+    /**
+     * Session-based guard for browser authentication.
+     */
+    web: sessionGuard({
+      /**
+       * Enable persistent login using remember-me tokens.
+       */
+      useRememberMeTokens: false,
+
+      provider: sessionUserProvider({
+        model: () => import('#models/user'),
+      }),
+    }),
+  },
+})
+
+export default authConfig
+
+/**
+ * Inferring types from the configured auth
+ * guards.
+ */
+declare module '@adonisjs/auth/types' {
+  export interface Authenticators extends InferAuthenticators<typeof authConfig> {}
+}
+declare module '@adonisjs/core/types' {
+  interface EventsList extends InferAuthEvents<Authenticators> {}
+}

config/bodyparser.ts

@@ -0,0 +1,78 @@
+import { defineConfig } from '@adonisjs/core/bodyparser'
+
+const bodyParserConfig = defineConfig({
+  /**
+   * Parse request bodies for these HTTP methods.
+   * Keep this aligned with methods that receive payloads in your routes.
+   */
+  allowedMethods: ['POST', 'PUT', 'PATCH', 'DELETE'],
+
+  /**
+   * Config for the "application/x-www-form-urlencoded"
+   * content-type parser.
+   */
+  form: {
+    /**
+     * Normalize empty string values to null.
+     */
+    convertEmptyStringsToNull: true,
+
+    /**
+     * Content types handled by the form parser.
+     */
+    types: ['application/x-www-form-urlencoded'],
+  },
+
+  /**
+   * Config for the JSON parser.
+   */
+  json: {
+    /**
+     * Normalize empty string values to null.
+     */
+    convertEmptyStringsToNull: true,
+
+    /**
+     * Content types handled by the JSON parser.
+     */
+    types: [
+      'application/json',
+      'application/json-patch+json',
+      'application/vnd.api+json',
+      'application/csp-report',
+    ],
+  },
+
+  /**
+   * Config for the "multipart/form-data" content-type parser.
+   * File uploads are handled by the multipart parser.
+   */
+  multipart: {
+    /**
+     * Automatically process uploaded files into the system tmp directory.
+     */
+    autoProcess: true,
+
+    /**
+     * Normalize empty string values to null.
+     */
+    convertEmptyStringsToNull: true,
+
+    /**
+     * Routes where multipart processing is handled manually.
+     */
+    processManually: [],
+
+    /**
+     * Maximum accepted payload size for multipart requests.
+     */
+    limit: '20mb',
+
+    /**
+     * Content types handled by the multipart parser.
+     */
+    types: ['multipart/form-data'],
+  },
+})
+
+export default bodyParserConfig

config/cors.ts

@@ -0,0 +1,50 @@
+import app from '@adonisjs/core/services/app'
+import { defineConfig } from '@adonisjs/cors'
+
+/**
+ * Configuration options to tweak the CORS policy. The following
+ * options are documented on the official documentation website.
+ *
+ * https://docs.adonisjs.com/guides/security/cors
+ */
+const corsConfig = defineConfig({
+  /**
+   * Enable or disable CORS handling globally.
+   */
+  enabled: true,
+
+  /**
+   * In development, allow every origin to simplify local front/backend setup.
+   * In production, keep an explicit allowlist (empty by default, so no
+   * cross-origin browser access is allowed until configured).
+   */
+  origin: app.inDev ? true : [],
+
+  /**
+   * HTTP methods accepted for cross-origin requests.
+   */
+  methods: ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'],
+
+  /**
+   * Reflect request headers by default. Use a string array to restrict
+   * allowed headers.
+   */
+  headers: true,
+
+  /**
+   * Response headers exposed to the browser.
+   */
+  exposeHeaders: [],
+
+  /**
+   * Allow cookies/authorization headers on cross-origin requests.
+   */
+  credentials: true,
+
+  /**
+   * Cache CORS preflight response for N seconds.
+   */
+  maxAge: 90,
+})
+
+export default corsConfig

config/database.ts

@@ -0,0 +1,138 @@
+import app from '@adonisjs/core/services/app'
+import { defineConfig } from '@adonisjs/lucid'
+import env from '#start/env'
+
+const dbConfig = defineConfig({
+  /**
+   * Default connection used for all queries.
+   */
+  connection: env.get('DB_CONNECTION', 'sqlite'),
+
+  connections: {
+    /**
+     * SQLite connection (default).
+     */
+    sqlite: {
+      client: 'better-sqlite3',
+
+      connection: {
+        filename: app.tmpPath('db.sqlite3'),
+      },
+
+      /**
+       * Required by Knex for SQLite defaults.
+       */
+      useNullAsDefault: true,
+
+      migrations: {
+        /**
+         * Sort migration files naturally by filename.
+         */
+        naturalSort: true,
+
+        /**
+         * Paths containing migration files.
+         */
+        paths: ['database/migrations'],
+      },
+
+      schemaGeneration: {
+        /**
+         * Enable schema generation from Lucid models.
+         */
+        enabled: true,
+
+        /**
+         * Custom schema rules file paths.
+         */
+        rulesPaths: ['./database/schema_rules.js'],
+      },
+    },
+
+    /**
+     * PostgreSQL connection.
+     * Install package to switch: npm install pg
+     */
+    pg: {
+      client: 'pg',
+      connection: {
+        host: env.get('DB_HOST'),
+        port: env.get('DB_PORT'),
+        user: env.get('DB_USER'),
+        password: env.get('DB_PASSWORD')?.release(),
+        database: env.get('DB_DATABASE'),
+        // For idempotent bulk writers (worker:fetch): don't wait for the WAL
+        // flush on every statement. A crash loses the last moments of writes,
+        // which the retried job writes again.
+        ...(env.get('DB_SYNCHRONOUS_COMMIT') === 'off' ? { options: '-c synchronous_commit=off' } : {}),
+      },
+      // One connection per concurrent job; knex defaults to 10.
+      pool: { min: 0, max: env.get('DB_POOL_MAX', 10) },
+      migrations: {
+        naturalSort: true,
+        paths: ['database/migrations'],
+      },
+      debug: app.inDev,
+    },
+
+    /**
+     * MySQL / MariaDB connection.
+     * Install package to switch: npm install mysql2
+     */
+    // mysql: {
+    //   client: 'mysql2',
+    //   connection: {
+    //     host: env.get('DB_HOST'),
+    //     port: env.get('DB_PORT'),
+    //     user: env.get('DB_USER'),
+    //     password: env.get('DB_PASSWORD'),
+    //     database: env.get('DB_DATABASE'),
+    //   },
+    //   migrations: {
+    //     naturalSort: true,
+    //     paths: ['database/migrations'],
+    //   },
+    //   debug: app.inDev,
+    // },
+
+    /**
+     * Microsoft SQL Server connection.
+     * Install package to switch: npm install tedious
+     */
+    // mssql: {
+    //   client: 'mssql',
+    //   connection: {
+    //     server: env.get('DB_HOST'),
+    //     port: env.get('DB_PORT'),
+    //     user: env.get('DB_USER'),
+    //     password: env.get('DB_PASSWORD'),
+    //     database: env.get('DB_DATABASE'),
+    //   },
+    //   migrations: {
+    //     naturalSort: true,
+    //     paths: ['database/migrations'],
+    //   },
+    //   debug: app.inDev,
+    // },
+
+    /**
+     * libSQL (Turso) connection.
+     * Install package to switch: npm install @libsql/client
+     */
+    // libsql: {
+    //   client: 'libsql',
+    //   connection: {
+    //     url: env.get('LIBSQL_URL'),
+    //     authToken: env.get('LIBSQL_AUTH_TOKEN'),
+    //   },
+    //   useNullAsDefault: true,
+    //   migrations: {
+    //     naturalSort: true,
+    //     paths: ['database/migrations'],
+    //   },
+    //   debug: app.inDev,
+    // },
+  },
+})
+
+export default dbConfig

config/encryption.ts

@@ -0,0 +1,34 @@
+import env from '#start/env'
+import { defineConfig, drivers } from '@adonisjs/core/encryption'
+
+const encryptionConfig = defineConfig({
+  /**
+   * Default encryption driver used by the application.
+   */
+  default: 'gcm',
+
+  list: {
+    gcm: drivers.aes256gcm({
+      /**
+       * Keys used for encryption/decryption.
+       * First key encrypts, all keys are tried for decryption.
+       */
+      keys: [env.get('APP_KEY')],
+
+      /**
+       * Stable identifier for this driver.
+       */
+      id: 'gcm',
+    }),
+  },
+})
+
+export default encryptionConfig
+
+/**
+ * Inferring types for the list of encryptors you have configured
+ * in your application.
+ */
+declare module '@adonisjs/core/types' {
+  export interface EncryptorsList extends InferEncryptors<typeof encryptionConfig> {}
+}

config/gc.ts

@@ -0,0 +1,77 @@
+import env from '#start/env'
+
+/**
+ * Retention for transient pipeline state in Redis. See CONCEPT.md (D11).
+ */
+export default {
+  /** How often `gc:scheduler` sweeps. */
+  intervalMs: env.get('GC_INTERVAL_MS', 3 * 60 * 1000),
+
+  jobs: {
+    /** Completed jobs are only kept for inspection; keep them briefly. */
+    completedAgeSeconds: env.get('GC_COMPLETED_AGE_SECONDS', 60 * 60),
+    completedMaxCount: 1000,
+    /** Failed jobs are kept longer so they can be inspected and retried. */
+    failedAgeSeconds: env.get('GC_FAILED_AGE_SECONDS', 7 * 24 * 60 * 60),
+  },
+
+  /**
+   * Upper bound on waiting body jobs. Obfuscated traffic can exceed what the
+   * line downloads for hours; beyond this, the oldest waiting jobs are
+   * dropped so Redis stays bounded (~1 KB per job).
+   */
+  bodyQueueMax: env.get('GC_BODY_QUEUE_MAX', 500_000),
+
+  binaries: {
+    /**
+     * A binary that gets no new segments for this long is deleted: incomplete
+     * ones are given up on, completed ones have been copied into `files`.
+     */
+    staleAfterSeconds: env.get('GC_BINARY_STALE_SECONDS', 48 * 60 * 60),
+    /**
+     * Header-derived binaries that still have one article after this long:
+     * uploads that give every article its own random name (and poster) never
+     * complete from headers. A real file gets its next articles in minutes.
+     */
+    singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60),
+    /**
+     * Completed header-derived binaries without a fingerprint (size, MD5 of
+     * the first 16 KiB): their file exists and no par2 set can name them
+     * later, so they go sooner than staleAfterSeconds.
+     */
+    unfingerprintedAfterSeconds: env.get('GC_UNFINGERPRINTED_SECONDS', 6 * 60 * 60),
+  },
+
+  /**
+   * Staged lone segments (CONCEPT.md D35) are dropped after this long. A
+   * real file's next segment comes in the same or the next fetch range,
+   * seconds to minutes later; at backfill speed two hours of waiting
+   * segments were 12M rows, 10 GB.
+   */
+  stagingTtlSeconds: env.get('STAGING_TTL_SECONDS', 30 * 60),
+
+  /** Releases without files or binaries (their binaries were collected) are deleted after this long. */
+  emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60),
+
+  /**
+   * Bulk deletes run in batches of this size, each kind (copied parts,
+   * single-article and unfingerprinted binaries, empty releases) for at most
+   * this long per sweep, so one backlog can't starve the others.
+   */
+  deleteBatch: 20_000,
+  deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 60 * 1000),
+  /** Bloated indexes are rebuilt at most this often (IndexMaintenance). */
+  reindexIntervalMs: env.get('GC_REINDEX_INTERVAL_MS', 7 * 24 * 60 * 60 * 1000),
+
+  /**
+   * Deferred files (CONCEPT.md D28) are deleted this long after they were
+   * first seen, processed or not. The articles stay on the server.
+   */
+  deferredRetentionDays: env.get('GC_DEFERRED_RETENTION_DAYS', 30),
+
+  /** Redis hashes from the old header worker (before D12); they only need to expire. */
+  legacyPendingFiles: {
+    prefix: 'file:',
+    ttlSeconds: 48 * 60 * 60,
+  },
+}

config/hash.ts

@@ -0,0 +1,75 @@
+import { defineConfig, drivers } from '@adonisjs/core/hash'
+
+/**
+ * Hashing configuration.
+ *
+ * This starter uses Node.js scrypt under the hood.
+ * Node.js reference: https://nodejs.org/api/crypto.html#cryptoscryptpassword-salt-keylen-options-callback
+ */
+const hashConfig = defineConfig({
+  /**
+   * Default hasher used by the application.
+   */
+  default: 'scrypt',
+
+  list: {
+    /**
+     * Scrypt is memory-hard, which makes brute-force attacks more expensive.
+     */
+    scrypt: drivers.scrypt({
+      /**
+       * Work factor (Node alias: N / cost).
+       * Higher values increase security and CPU+memory usage.
+       *
+       * Tuning guideline:
+       * - Start with 16384.
+       * - Increase gradually (for example 32768) and benchmark login/signup latency.
+       * - Keep values practical for your slowest production machine.
+       *
+       * Node constraint: value must be a power of two greater than 1.
+       */
+      cost: 16384,
+
+      /**
+       * Block size (Node alias: r / blockSize).
+       * Increases memory and CPU linearly.
+       *
+       * Tuning guideline:
+       * - Keep 8 unless you have a measured reason to change it.
+       * - Raise only with benchmark data, because memory usage grows quickly.
+       */
+      blockSize: 8,
+
+      /**
+       * Parallelization (Node alias: p / parallelization).
+       * Controls how many independent computations are performed.
+       *
+       * Tuning guideline:
+       * - Keep 1 for most applications.
+       * - Increase only after load testing if your infrastructure benefits from it.
+       */
+      parallelization: 1,
+
+      /**
+       * Maximum memory limit in bytes (Node alias: maxmem / maxMemory).
+       * Hashing throws if the estimated memory usage is above this limit.
+       * Node documents the check as approximately: 128 * N * r > maxmem.
+       *
+       * Tuning guideline:
+       * - Keep this aligned with your cost/blockSize choices.
+       * - Increase carefully on memory-constrained environments.
+       */
+      maxMemory: 33554432,
+    }),
+  },
+})
+
+export default hashConfig
+
+/**
+ * Inferring types for the list of hashers you have configured
+ * in your application.
+ */
+declare module '@adonisjs/core/types' {
+  export interface HashersList extends InferHashers<typeof hashConfig> {}
+}

config/indexer.ts

@@ -0,0 +1,84 @@
+import env from '#start/env'
+
+export default {
+  /** How often `index:scheduler` checks the groups for new articles. */
+  intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
+  /** Articles per fetch job (one XOVER call). */
+  batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
+  /** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
+  maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
+  /**
+   * Where a group that was never indexed starts: this many articles before
+   * the newest one. The server's low-water mark can be years back.
+   */
+  initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),
+
+  /**
+   * Obfuscated articles whose message-id matches one of these are not sent
+   * to the body queue: their uploader spreads every file over many groups,
+   * so identifying them would take far more bandwidth than there is
+   * (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
+   */
+  skipBodyMessageIds: [
+    // hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
+    /^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
+  ],
+
+  /**
+   * Obfuscated articles whose message-id matches one of these don't go to
+   * the body queue. Instead they're recorded per file in `deferred_files`
+   * (their subject keeps the file name, only the part counter is missing).
+   * `worker:deferred` later fetches one body per file (CONCEPT.md D28).
+   */
+  deferBodyMessageIds: [
+    // ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
+    /^<?[0-9a-f]{32}@ngPost>?$/,
+  ],
+  /**
+   * Spotnet header groups (CONCEPT.md D29): their headers are spots, not
+   * binaries. Add one with `group:add free.pt --backfill 5000`.
+   */
+  spotGroups: ['free.pt'],
+  /**
+   * Spots posted within this many days are imported (their NZB read and its
+   * files added). Older spots are listed: title, size and date, with the NZB
+   * read on first download.
+   */
+  spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),
+
+  /**
+   * Groups whose lone header segments wait in `staged_segments` until their
+   * file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
+   */
+  stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
+
+  /**
+   * Groups whose staged segments also group by poster, file counter `[i/M]`
+   * and part count (CONCEPT.md D36), for uploads with a random subject per
+   * article. Needs staging. Comma-separated; `*` for all.
+   */
+  counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
+
+  /**
+   * Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
+   * header-only fetch ranges walking back through a group's history.
+   */
+  backfill: {
+    /** Backfill ranges waiting or running at a time, over all groups. */
+    maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
+    /** BullMQ priority (higher runs later; live ranges have none, so they go first). */
+    priority: 10,
+    /**
+     * No new backfill ranges while the database's NVMe part (default
+     * tablespace) or its HDD part (tablespace `hdd`, the files table) is
+     * larger than this. Checked every scheduler tick.
+     */
+    maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
+    maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
+  },
+
+  /** worker:deferred leaves a file alone until it has had no new articles for this long. */
+  deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
+  /** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
+  deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
+}

config/logger.ts

@@ -0,0 +1,51 @@
+import env from '#start/env'
+import app from '@adonisjs/core/services/app'
+import { defineConfig, syncDestination, targets } from '@adonisjs/core/logger'
+
+const loggerConfig = defineConfig({
+  /**
+   * Default logger name used by ctx.logger and app logger calls.
+   */
+  default: 'app',
+
+  loggers: {
+    app: {
+      /**
+       * Toggle this logger on/off.
+       */
+      enabled: true,
+
+      /**
+       * Logger name shown in log records.
+       */
+      name: env.get('APP_NAME'),
+
+      /**
+       * Minimum level to output (trace, debug, info, warn, error, fatal).
+       */
+      level: env.get('LOG_LEVEL'),
+
+      /**
+       * Use sync destination in non-production for immediate flush.
+       */
+      destination: !app.inProduction ? await syncDestination() : undefined,
+
+      /**
+       * Configure where logs are written.
+       */
+      transport: {
+        targets: [targets.file({ destination: 1 })],
+      },
+    },
+  },
+})
+
+export default loggerConfig
+
+/**
+ * Inferring types for the list of loggers you have configured
+ * in your application.
+ */
+declare module '@adonisjs/core/types' {
+  export interface LoggersList extends InferLoggers<typeof loggerConfig> {}
+}

config/nntp.ts

@@ -0,0 +1,13 @@
+import env from '#start/env'
+
+export default {
+  host: env.get('NNTP_HOST'),
+  port: env.get('NNTP_PORT', 119),
+  user: env.get('NNTP_USER'),
+  password: env.get('NNTP_PASSWORD'),
+  secure: env.get('NNTP_SECURE', false),
+  /** Connections per process. Mind the provider's limit across all processes. */
+  poolSize: env.get('NNTP_POOL_SIZE', 10),
+  /** A command without an answer this long counts as a dead connection (XOVER of 20k headers takes ~0.2 s). */
+  commandTimeoutMs: env.get('NNTP_COMMAND_TIMEOUT_MS', 120_000),
+}

config/queue.ts

@@ -0,0 +1,8 @@
+import env from '#start/env'
+
+export default {
+  connection: {
+    host: env.get('REDIS_HOST', '127.0.0.1'),
+    port: env.get('REDIS_PORT', 6379),
+  },
+}

config/redis.ts

@@ -0,0 +1,12 @@
+import env from '#start/env'
+
+export default {
+  connection: 'main',
+  connections: {
+    main: {
+      host: env.get('REDIS_HOST', '127.0.0.1'),
+      port: env.get('REDIS_PORT'),
+            password: env.get('REDIS_PASSWORD')?.release(),
+          },
+  },
+}

config/session.ts

@@ -0,0 +1,78 @@
+import env from '#start/env'
+import app from '@adonisjs/core/services/app'
+import { defineConfig, stores } from '@adonisjs/session'
+
+const sessionConfig = defineConfig({
+  /**
+   * Enable or disable session support globally.
+   */
+  enabled: true,
+
+  /**
+   * Cookie name storing the session identifier.
+   */
+  cookieName: 'adonis-session',
+
+  /**
+   * When set to true, the session id cookie will be deleted
+   * once the user closes the browser.
+   */
+  clearWithBrowser: false,
+
+  /**
+   * Define how long to keep the session data alive without
+   * any activity.
+   */
+  age: '2h',
+
+  /**
+   * Configuration for session cookie and the
+   * cookie store.
+   */
+  cookie: {
+    /**
+     * Restrict the cookie to a URL path. '/' means all routes.
+     */
+    path: '/',
+
+    /**
+     * Prevent JavaScript access to the cookie in the browser.
+     */
+    httpOnly: true,
+
+    /**
+     * Send cookies only over HTTPS in production.
+     */
+    secure: app.inProduction,
+
+    /**
+     * Cross-site policy for cookie sending.
+     */
+    sameSite: 'lax',
+  },
+
+  /**
+   * The store to use. Make sure to validate the environment
+   * variable in order to infer the store name without any
+   * errors.
+   */
+  store: env.get('SESSION_DRIVER'),
+
+  /**
+   * List of configured stores. Refer documentation to see
+   * list of available stores and their config.
+   */
+  stores: {
+    /**
+     * Store session data inside encrypted cookies.
+     */
+    cookie: stores.cookie(),
+
+    /**
+     * Store session data inside the configured database.
+     */
+    database: stores.database(),
+  },
+})
+
+export default sessionConfig

config/shield.ts

@@ -0,0 +1,95 @@
+import { defineConfig } from '@adonisjs/shield'
+
+const shieldConfig = defineConfig({
+  /**
+   * Configure CSP policies for your app. Refer documentation
+   * to learn more.
+   */
+  csp: {
+    /**
+     * Enable the Content-Security-Policy header.
+     */
+    enabled: false,
+
+    /**
+     * Per-resource CSP directives.
+     */
+    directives: {},
+
+    /**
+     * Report violations without blocking resources.
+     */
+    reportOnly: false,
+  },
+
+  /**
+   * Configure CSRF protection options. Refer documentation
+   * to learn more.
+   */
+  csrf: {
+    /**
+     * Enable CSRF token verification for state-changing requests.
+     */
+    enabled: false,
+
+    /**
+     * Route patterns to exclude from CSRF checks.
+     * Useful for external webhooks or API endpoints.
+     */
+    exceptRoutes: [],
+
+    /**
+     * Expose an encrypted XSRF-TOKEN cookie for frontend HTTP clients.
+     */
+    enableXsrfCookie: true,
+
+    /**
+     * HTTP methods protected by CSRF validation.
+     */
+    methods: ['POST', 'PUT', 'PATCH', 'DELETE'],
+  },
+
+  /**
+   * Control how your website should be embedded inside
+   * iframes.
+   */
+  xFrame: {
+    /**
+     * Enable the X-Frame-Options header.
+     */
+    enabled: true,
+
+    /**
+     * Block all framing attempts. Default value is DENY.
+     */
+    action: 'DENY',
+  },
+
+  /**
+   * Force browser to always use HTTPS.
+   */
+  hsts: {
+    /**
+     * Enable the Strict-Transport-Security header.
+     */
+    enabled: true,
+
+    /**
+     * HSTS policy duration remembered by browsers.
+     */
+    maxAge: '180 days',
+  },
+
+  /**
+   * Disable browsers from sniffing content types and rely only
+   * on the response content-type header.
+   */
+  contentTypeSniffing: {
+    /**
+     * Enable X-Content-Type-Options: nosniff.
+     */
+    enabled: true,
+  },
+})
+
+export default shieldConfig

config/static.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from '@adonisjs/static'
+
+/**
+ * Configuration options to tweak the static files middleware.
+ * The complete set of options are documented on the
+ * official documentation website.
+ *
+ * https://docs.adonisjs.com/guides/static-assets
+ */
+const staticServerConfig = defineConfig({
+  enabled: true,
+  etag: true,
+  lastModified: true,
+  dotFiles: 'ignore',
+})
+
+export default staticServerConfig