usenet-indexer/config/indexer.ts

import env from '#start/env'

export default {
  /** How often `index:scheduler` checks the groups for new articles. */
  intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
  /** Articles per fetch job (one XOVER call). */
  batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
  /** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
  maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
  /**
   * Where a group that was never indexed starts: this many articles before
   * the newest one. The server's low-water mark can be years back.
   */
  initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),

  /**
   * Obfuscated articles whose message-id matches one of these are not sent
   * to the body queue: their uploader spreads every file over many groups,
   * so identifying them would take far more bandwidth than there is
   * (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
   */
  skipBodyMessageIds: [
    // hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
    /^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
  ],

  /**
   * Obfuscated articles whose message-id matches one of these don't go to
   * the body queue. Instead they're recorded per file in `deferred_files`
   * (their subject keeps the file name, only the part counter is missing).
   * `worker:deferred` later fetches one body per file (CONCEPT.md D28).
   */
  deferBodyMessageIds: [
    // ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
    /^<?[0-9a-f]{32}@ngPost>?$/,
  ],
  /**
   * Spotnet header groups (CONCEPT.md D29): their headers are spots, not
   * binaries. Add one with `group:add free.pt --backfill 5000`.
   */
  spotGroups: ['free.pt'],
  /**
   * Spots posted within this many days are imported (their NZB read and its
   * files added). Older spots are listed: title, size and date, with the NZB
   * read on first download.
   */
  spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),

  /**
   * Groups whose lone header segments wait in `staged_segments` until their
   * file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
   */
  stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),

  /**
   * Groups whose staged segments also group by poster, file counter `[i/M]`
   * and part count (CONCEPT.md D36), for uploads with a random subject per
   * article. Needs staging. Comma-separated; `*` for all.
   */
  counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),

  /**
   * Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
   * header-only fetch ranges walking back through a group's history.
   */
  seed: {
    /** NNTP connections the seed API fetches with for relay nodes (CONCEPT.md D38); 0: relay off. */
    nntpConnections: env.get('SEED_NNTP_CONNECTIONS', 0),
    /**
     * Silence that counts as a dead connection during a relay XOVER (an
     * inactivity timeout: slow ranges keep sending). Chunks that "never
     * answered" in 10 minutes came in 0.3 s on a fresh connection.
     */
    nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 90 * 1000),
    /**
     * Ask the news server for gzip-compressed overviews (SEED_NNTP_COMPRESS=false: plain).
     * On one connection plain had a shorter tail for 2026 articles, but with 20 relay
     * connections both gave ~1.5 chunks/s, plain at many times the relay's CPU and
     * more traffic (LOG.md 2026-10-06 (9)).
     */
    nntpCompress: env.get('SEED_NNTP_COMPRESS', true),
  },

  backfill: {
    /** Backfill ranges waiting or running at a time, over all groups. */
    maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
    /** BullMQ priority (higher runs later; live ranges have none, so they go first). */
    priority: 10,
    /**
     * No new backfill ranges while the database's NVMe part (default
     * tablespace) or its HDD part (tablespace `hdd`, the files table) is
     * larger than this. Checked every scheduler tick.
     */
    maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
    maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
  },

  /**
   * Dropping garbage uploads early (CONCEPT.md D42). A signature (posting tool, first
   * group) is dropped once at least minProbes of its probed random-named releases are
   * known and at most maxNamedRate of them got a real name or password; its
   * random-named files are then not stored, except explorePercent of its releases.
   */
  garbage: {
    /** Off: nothing is filtered or deleted (the learning still runs). */
    enabled: env.get('GARBAGE_FILTER', true),
    explorePercent: env.get('GARBAGE_EXPLORE_PERCENT', 2),
    minProbes: 100,
    maxNamedRate: 0.005,
    /** Probed releases learned from: 1 in this many (by release id), for the last windowDays. */
    probeSample: 4,
    windowDays: 30,
    /** How often gc:scheduler learns the signatures again. */
    learnIntervalMs: env.get('GARBAGE_LEARN_INTERVAL_MS', 3 * 60 * 60 * 1000),
  },

  /** worker:deferred leaves a file alone until it has had no new articles for this long. */
  deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
  /** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
  deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
}