implemented a lot of features

config/database.ts

@@ -61,7 +61,13 @@ const dbConfig = defineConfig({
         user: env.get('DB_USER'),
         password: env.get('DB_PASSWORD')?.release(),
         database: env.get('DB_DATABASE'),
+        // For idempotent bulk writers (worker:fetch): don't wait for the WAL
+        // flush on every statement. A crash loses the last moments of writes,
+        // which the retried job writes again.
+        ...(env.get('DB_SYNCHRONOUS_COMMIT') === 'off' ? { options: '-c synchronous_commit=off' } : {}),
       },
+      // One connection per concurrent job; knex defaults to 10.
+      pool: { min: 0, max: env.get('DB_POOL_MAX', 10) },
       migrations: {
         naturalSort: true,
         paths: ['database/migrations'],

config/gc.ts

@@ -0,0 +1,57 @@
+import env from '#start/env'
+
+/**
+ * Retention for transient pipeline state in Redis. See CONCEPT.md (D11).
+ */
+export default {
+  /** How often `gc:scheduler` sweeps. */
+  intervalMs: env.get('GC_INTERVAL_MS', 10 * 60 * 1000),
+
+  jobs: {
+    /** Completed jobs are only kept for inspection; keep them briefly. */
+    completedAgeSeconds: env.get('GC_COMPLETED_AGE_SECONDS', 60 * 60),
+    completedMaxCount: 1000,
+    /** Failed jobs are kept longer so they can be inspected and retried. */
+    failedAgeSeconds: env.get('GC_FAILED_AGE_SECONDS', 7 * 24 * 60 * 60),
+  },
+
+  /**
+   * Upper bound on waiting body jobs. Obfuscated traffic can exceed what the
+   * line downloads for hours; beyond this, the oldest waiting jobs are
+   * dropped so Redis stays bounded (~1 KB per job).
+   */
+  bodyQueueMax: env.get('GC_BODY_QUEUE_MAX', 500_000),
+
+  binaries: {
+    /**
+     * A binary that gets no new segments for this long is deleted: incomplete
+     * ones are given up on, completed ones have been copied into `files`.
+     */
+    staleAfterSeconds: env.get('GC_BINARY_STALE_SECONDS', 48 * 60 * 60),
+    /**
+     * Header-derived binaries that still have one article after this long:
+     * uploads that give every article its own random name (and poster) never
+     * complete from headers. A real file gets its next articles in minutes.
+     */
+    singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60),
+  },
+
+  /** Releases without files or binaries (their binaries were collected) are deleted after this long. */
+  emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60),
+
+  /** Bulk deletes run in batches of this size, for at most this long per sweep. */
+  deleteBatch: 20_000,
+  deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 3 * 60 * 1000),
+
+  /**
+   * Deferred files (CONCEPT.md D28) are deleted this long after they were
+   * first seen, processed or not. The articles stay on the server.
+   */
+  deferredRetentionDays: env.get('GC_DEFERRED_RETENTION_DAYS', 30),
+
+  /** Redis hashes from the old header worker (before D12); they only need to expire. */
+  legacyPendingFiles: {
+    prefix: 'file:',
+    ttlSeconds: 48 * 60 * 60,
+  },
+}

config/indexer.ts

@@ -0,0 +1,47 @@
+import env from '#start/env'
+
+export default {
+  /** How often `index:scheduler` checks the groups for new articles. */
+  intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
+  /** Articles per fetch job (one XOVER call). */
+  batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
+  /** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
+  maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
+  /**
+   * Where a group that was never indexed starts: this many articles before
+   * the newest one. The server's low-water mark can be years back.
+   */
+  initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),
+
+  /**
+   * Obfuscated articles whose message-id matches one of these are not sent
+   * to the body queue: their uploader spreads every file over many groups,
+   * so identifying them would take far more bandwidth than there is
+   * (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
+   */
+  skipBodyMessageIds: [
+    // hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
+    /^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
+  ],
+
+  /**
+   * Obfuscated articles whose message-id matches one of these don't go to
+   * the body queue. Instead they're recorded per file in `deferred_files`
+   * (their subject keeps the file name, only the part counter is missing).
+   * `worker:deferred` later fetches one body per file (CONCEPT.md D28).
+   */
+  deferBodyMessageIds: [
+    // ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
+    /^<?[0-9a-f]{32}@ngPost>?$/,
+  ],
+  /**
+   * Spotnet header groups (CONCEPT.md D29): their headers are spots, not
+   * binaries. Add one with `group:add free.pt --backfill 5000`.
+   */
+  spotGroups: ['free.pt'],
+
+  /** worker:deferred leaves a file alone until it has had no new articles for this long. */
+  deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
+  /** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
+  deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
+}

config/nntp.ts

@@ -6,4 +6,6 @@ export default {
   user: env.get('NNTP_USER'),
   password: env.get('NNTP_PASSWORD'),
   secure: env.get('NNTP_SECURE', false),
+  /** Connections per process. Mind the provider's limit across all processes. */
+  poolSize: env.get('NNTP_POOL_SIZE', 10),
 }

config/static.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from '@adonisjs/static'
+
+/**
+ * Configuration options to tweak the static files middleware.
+ * The complete set of options are documented on the
+ * official documentation website.
+ *
+ * https://docs.adonisjs.com/guides/static-assets
+ */
+const staticServerConfig = defineConfig({
+  enabled: true,
+  etag: true,
+  lastModified: true,
+  dotFiles: 'ignore',
+})
+
+export default staticServerConfig