Daan MeijerandClaude Opus 5.5
530a752cca
- A range reported to its end but not finished (its node went away at the very end) was leased again with start past end; nodes asked for "from 49208000000 to 49207999999", got 400, and the range failed after five leases. The orchestrator now marks such ranges done before leasing, and runRange reports them done without fetching. - Some old ranges come so slowly that one XOVER of 20k headers takes over two minutes: five ranges failed on "NNTP reply timed out". The relay pool's timeout is SEED_NNTP_TIMEOUT_MS (default 10 minutes, the node's HTTP timeout). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
92 lines
4.0 KiB
TypeScript
92 lines
4.0 KiB
TypeScript
import env from '#start/env'
|
|
|
|
export default {
|
|
/** How often `index:scheduler` checks the groups for new articles. */
|
|
intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
|
|
/** Articles per fetch job (one XOVER call). */
|
|
batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
|
|
/** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
|
|
maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
|
|
/**
|
|
* Where a group that was never indexed starts: this many articles before
|
|
* the newest one. The server's low-water mark can be years back.
|
|
*/
|
|
initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),
|
|
|
|
/**
|
|
* Obfuscated articles whose message-id matches one of these are not sent
|
|
* to the body queue: their uploader spreads every file over many groups,
|
|
* so identifying them would take far more bandwidth than there is
|
|
* (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
|
|
*/
|
|
skipBodyMessageIds: [
|
|
// hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
|
|
/^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
|
|
],
|
|
|
|
/**
|
|
* Obfuscated articles whose message-id matches one of these don't go to
|
|
* the body queue. Instead they're recorded per file in `deferred_files`
|
|
* (their subject keeps the file name, only the part counter is missing).
|
|
* `worker:deferred` later fetches one body per file (CONCEPT.md D28).
|
|
*/
|
|
deferBodyMessageIds: [
|
|
// ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
|
|
/^<?[0-9a-f]{32}@ngPost>?$/,
|
|
],
|
|
/**
|
|
* Spotnet header groups (CONCEPT.md D29): their headers are spots, not
|
|
* binaries. Add one with `group:add free.pt --backfill 5000`.
|
|
*/
|
|
spotGroups: ['free.pt'],
|
|
/**
|
|
* Spots posted within this many days are imported (their NZB read and its
|
|
* files added). Older spots are listed: title, size and date, with the NZB
|
|
* read on first download.
|
|
*/
|
|
spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),
|
|
|
|
/**
|
|
* Groups whose lone header segments wait in `staged_segments` until their
|
|
* file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
|
|
*/
|
|
stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
|
|
|
|
/**
|
|
* Groups whose staged segments also group by poster, file counter `[i/M]`
|
|
* and part count (CONCEPT.md D36), for uploads with a random subject per
|
|
* article. Needs staging. Comma-separated; `*` for all.
|
|
*/
|
|
counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
|
|
|
|
/**
|
|
* Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
|
|
* header-only fetch ranges walking back through a group's history.
|
|
*/
|
|
seed: {
|
|
/** NNTP connections the seed API fetches with for relay nodes (CONCEPT.md D38); 0: relay off. */
|
|
nntpConnections: env.get('SEED_NNTP_CONNECTIONS', 0),
|
|
/** A relay XOVER may take this long: some old ranges come at ~1k headers/s (and slower when cold). */
|
|
nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 10 * 60 * 1000),
|
|
},
|
|
|
|
backfill: {
|
|
/** Backfill ranges waiting or running at a time, over all groups. */
|
|
maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
|
|
/** BullMQ priority (higher runs later; live ranges have none, so they go first). */
|
|
priority: 10,
|
|
/**
|
|
* No new backfill ranges while the database's NVMe part (default
|
|
* tablespace) or its HDD part (tablespace `hdd`, the files table) is
|
|
* larger than this. Checked every scheduler tick.
|
|
*/
|
|
maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
|
|
maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
|
|
},
|
|
|
|
/** worker:deferred leaves a file alone until it has had no new articles for this long. */
|
|
deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
|
|
/** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
|
|
deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
|
|
}
|