implemented a lot of features

config/indexer.ts

@@ -0,0 +1,47 @@
+import env from '#start/env'
+
+export default {
+  /** How often `index:scheduler` checks the groups for new articles. */
+  intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
+  /** Articles per fetch job (one XOVER call). */
+  batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
+  /** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
+  maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
+  /**
+   * Where a group that was never indexed starts: this many articles before
+   * the newest one. The server's low-water mark can be years back.
+   */
+  initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),
+
+  /**
+   * Obfuscated articles whose message-id matches one of these are not sent
+   * to the body queue: their uploader spreads every file over many groups,
+   * so identifying them would take far more bandwidth than there is
+   * (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
+   */
+  skipBodyMessageIds: [
+    // hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
+    /^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
+  ],
+
+  /**
+   * Obfuscated articles whose message-id matches one of these don't go to
+   * the body queue. Instead they're recorded per file in `deferred_files`
+   * (their subject keeps the file name, only the part counter is missing).
+   * `worker:deferred` later fetches one body per file (CONCEPT.md D28).
+   */
+  deferBodyMessageIds: [
+    // ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
+    /^<?[0-9a-f]{32}@ngPost>?$/,
+  ],
+  /**
+   * Spotnet header groups (CONCEPT.md D29): their headers are spots, not
+   * binaries. Add one with `group:add free.pt --backfill 5000`.
+   */
+  spotGroups: ['free.pt'],
+
+  /** worker:deferred leaves a file alone until it has had no new articles for this long. */
+  deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
+  /** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
+  deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
+}