usenet-indexer/config/coordinator.ts

import env from '#start/env'
import app from '@adonisjs/core/services/app'

/** The coordinator (REBUILD.md, part 1). */
export default {
  /** Articles per range. */
  rangeArticles: env.get('RANGE_ARTICLES', 2_000_000),
  /**
   * Articles per XOVER chunk (spool files, stream frames). Eweka takes
   * ~1.5–2 s per command whatever its size: one connection read 9–15k
   * lines/s in chunks of 25k, ~20k in 50k, ~30k in 100k and 200k (LOG.md
   * 2026-10-09).
   */
  chunkArticles: env.get('CHUNK_ARTICLES', 100_000),
  /**
   * The live tail (REBUILD.md, coordinator): new articles become ranges
   * every `everySeconds` (one LIST ACTIVE per name prefix), leased and
   * fetched before any history. A group gets a tail range once it has
   * `minArticles` new (a chunk) or its oldest unplanned article waited
   * `maxDelayMinutes`, so a quiet group doesn't make a range every round.
   */
  tail: {
    everySeconds: env.get('TAIL_EVERY_SECONDS', 120),
    minArticles: env.get('TAIL_MIN_ARTICLES', 100_000),
    maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15),
    /**
     * Before everything: the history's 0 (oldest) … 1000 (newest), the stored ranges' -2 and ranges raised to -1 (the
     * short gaps between done blocks went before the tail at -1 and it fell ~4 h behind; LOG.md 2026-10-16 (70)).
     */
    priority: -3,
  },
  /** A leased range without a heartbeat for this long goes back to the queue. */
  leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
  /**
   * The ranges group leases hand out (REBUILD.md design 1), by priority: the tail is negative, the capture windows
   * 0–2 (2016, 2021, 2025), the history by its position in the group (0 oldest … 1000 newest: its first buckets
   * share 0–2 with the windows) or 4000+. Bounds so a deployment can run one window first, with
   * LEASE_CAPTURE_ONLY so the history's 0 isn't taken along (homelab 2026-10-26: 0..0 and capture only, the 2016
   * window; LOG.md (118)); unbounded and any range by default.
   */
  leasePriorityMin: env.get('LEASE_MIN_PRIORITY', -1_000_000),
  leasePriorityMax: env.get('LEASE_MAX_PRIORITY', 1_000_000),
  leaseCaptureOnly: env.get('LEASE_CAPTURE_ONLY', false),
  /** NNTP connections the coordinator fetches headers with: at most (its share, nntpShares), and at least while sharing. */
  nntpConnections: env.get('NNTP_CONNECTIONS', 4),
  nntpMinConnections: env.get('NNTP_MIN_CONNECTIONS', 8),
  /**
   * The account's connections, shared by the coordinator and the prober by priority (app/services/nntp_shares.ts):
   * all of them less the reserved ones (the spotter's and the spot inspector's 1 each, the coordinator's plain one,
   * and 2 for a hedged request's or a reconnect's new connection while the old one closes).
   */
  nntpShares: {
    total: env.get('NNTP_ACCOUNT_CONNECTIONS', 50) - env.get('NNTP_RESERVED_CONNECTIONS', 5),
    everyMs: 15_000,
    /** Headers before probes: history takes what it can use, the prober the rest. */
    coordinatorPriority: 2,
    proberPriority: 1,
  },
  /**
   * A reply with no data for this long fails, and its connection is replaced.
   * Eweka sometimes stops answering on long-lived connections (all 40 quiet,
   * a fresh one answers in seconds; LOG.md 2026-10-09 (14)): at 10 minutes
   * the prefetcher nearly stopped. Data arrives within seconds otherwise.
   */
  nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 2 * 60 * 1000),
  /**
   * A request that got less than this in the last minute (kB) is given up,
   * its connection closed, and the chunk fetched again: a reply that trickles
   * (6,204 → 6,229 kB in a minute) never goes quiet long enough for
   * nntpTimeoutMs, and kept a worker waiting on one chunk for an hour. Not a
   * deadline: some groups send steadily but slowly (a.b.0day.stuffz ~400 kB a
   * minute, 5–7 minutes a chunk), and a fixed 90–360 s gave those up again
   * and again (LOG.md 2026-10-10 (3)).
   */
  nntpStallBytesPerMinute: env.get('NNTP_STALL_KB_PER_MINUTE', 100) * 1000,
  /**
   * A chunk not fetched after this long is asked again on another
   * connection; the first answer wins and the other connection is closed.
   * Eweka stalls single requests: the same chunk took 27–52 s on one
   * connection and 1–9 s on another at the same moment; median ~3 s
   * (LOG.md 2026-10-09 (15)). Measured in production (40 + 6 connections vs
   * 40, 15 min each): 280k vs 273k articles/s, within the noise; the second
   * request won half the time. Off by default.
   */
  nntpHedgeAfterMs: env.get('NNTP_HEDGE_AFTER_SECONDS', 10) * 1000,
  /** Of nntpConnections, kept for those second requests (the prefetcher's loops are the rest). 0: no hedging. */
  nntpHedgeConnections: env.get('NNTP_HEDGE_CONNECTIONS', 0),
  /**
   * A connection is closed after this long (when it's free), and a fresh one
   * opened. 0: never. Requests were as often slow (>10 s) on connections
   * under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)).
   */
  nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000,
  /**
   * A connection with nothing to do for this long is closed, and opened again
   * when there's work. 0: kept. Eweka drops connections that stay quiet, and
   * the next command on one failed ("Spot inspection failed: NNTP connection
   * closed"); idle ones also take the account's connections from downloads.
   */
  nntpIdleCloseMs: env.get('NNTP_IDLE_CLOSE_SECONDS', 60) * 1000,
  probe: {
    /**
     * At most (its share, nntpShares): NNTP connections the prober reads articles with, besides the
     * coordinator's: the account has 50, the coordinator 40 + 1 plain, and a
     * couple stay free for downloads. ~7 MB/s each (~10 articles/s).
     */
    connections: env.get('PROBE_CONNECTIONS', 6),
    /** Of them, always (its share's minimum; nntpShares). */
    minConnections: env.get('PROBE_MIN_CONNECTIONS', 2),
    /** No claims while the filesystem under `/` (the NVMe, in the container) has less free (app/services/disk_space.ts). 0: off. */
    minFreeGb: env.get('PROBE_MIN_FREE_GB', 0),
    /** Candidates claimed at a time. */
    batch: 400,
    /** BODY commands in flight per connection (pipelined; XoverClient.bodies). */
    pipeline: env.get('PROBE_PIPELINE', 8),
    /**
     * The pending ladder (REBUILD.md decision f; app/services/pending_ladder.ts): the share of each claim taken
     * from pending_releases at least (the catalog's own candidates come first, the rest of the claim is theirs).
     */
    pendingShare: env.get('PENDING_SHARE', 0.25),
    /** A signature (tool, group, poster) with this many climbs and at most pendingMaxNamed of them named is dropped. */
    pendingMinProbes: env.get('PENDING_MIN_PROBES', 100),
    pendingMaxNamed: env.get('PENDING_MAX_NAMED', 0.005),
  },
  /**
   * The catalog's tablespace may grow to this many GB (app/services/catalog_space.ts); above it only the tail and
   * the stored ranges are leased and prefetched. 0: no cap.
   */
  catalogMaxNvmeGb: env.get('CATALOG_MAX_NVME_GB', 0),
  /**
   * Spotnet spots (app/services/spotter.ts, spot_inspector.ts): the group the spots are posted in, how many of
   * its last articles the spotter reads the first time (~215 spots a day: ~3 months), how often it looks again,
   * the most articles an NZB may take, and how long an inspected spot whose files match no release waits for
   * the header pass before it becomes a release of its own.
   */
  spots: {
    group: env.get('SPOT_GROUP', 'free.pt'),
    backfill: env.get('SPOT_BACKFILL', 20_000),
    everySeconds: env.get('SPOT_EVERY_SECONDS', 120),
    maxNzbSegments: env.get('SPOT_MAX_NZB_SEGMENTS', 64),
    standaloneAfterHours: env.get('SPOT_STANDALONE_AFTER_HOURS', 24),
  },
  /** Stored XOVER chunks of ranges marked `capture`, for the grouping sweeps (app/services/corpus.ts). Off when unset. */
  corpus: {
    dir: env.get('CORPUS_DIR'),
    /**
     * Capture ranges fetched into the corpus by connections the spool leaves idle, this many at a time (8, 2 connections each, left 20 of 40 idle): the
     * spool only fetches what workers lease soon, and the tail comes first, so after a reset the capture
     * ranges waited behind it for hours (1,400 marked, none stored, 34 of 40 connections idle; LOG.md 2026-10-12 (27)).
     */
    ahead: env.get('CORPUS_AHEAD', 16),
    /**
     * Stored without the lines the edge worker drops on their own (crossposts another indexed group owns, obfuscated
     * subjects; ~48% of the indexed ranges' headers), what was left out in each chunk's `.drops`: twice the history
     * in the same space (LOG.md 2026-10-23 (101)). Off: as the news server sent them.
     */
    filter: env.get('CORPUS_FILTER', true),
  },
  /**
   * The group windows' snapshots (REBUILD.md, "Voyager 3: window indexing", 5; app/services/group_leases.ts): a
   * file a group on bulk, next to the spool by default.
   */
  snapshotDir: env.get('SNAPSHOT_DIR', `${env.get('SPOOL_DIR', app.tmpPath('spool'))}/snapshots`),
  spool: {
    dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
    /** Ranges kept fetched ahead, per active lease. */
    aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 4),
    /** At least this many ranges ahead, so the first leases start at once. */
    minAhead: env.get('SPOOL_MIN_AHEAD', 2),
    maxBytes: env.get('SPOOL_MAX_GB', 10) * 1e9,
    /**
     * At most this many connections on one range, and on one group, at a
     * time (0: no limit): spreads the requests over ranges and groups instead
     * of all on the next chunks of the first ranges. Homelab, 40 connections,
     * 12 min each, alternated: unlimited (on ~10 ranges) 281k and 167k
     * articles/s, 2 per range and 4 per group (on ~22) 319k and 296k, with
     * shorter chunk times (11.5 vs 13–24 s; LOG.md 2026-10-09 (18)). Needs
     * targets for it: aheadPerLease 4.
     */
    perRange: env.get('PREFETCH_PER_RANGE', 2),
    perGroup: env.get('PREFETCH_PER_GROUP', 4),
  },
}