Daan MeijerandClaude Opus 5.5
5ba93f558c
Eweka takes ~1.5–2 s per XOVER command whatever its size. One connection on recent a.b.boneless: 9–15k lines/s in chunks of 25k, ~20k in 50k, ~30k in 100k–200k. Four connections at once: 15k each at 50k, 22k each at 100k (the same as one connection alone at that moment): connections don't slow each other. CHUNK_ARTICLES defaults to 100k; edge:node posts a batch every --batch-articles (500k) instead of every N chunks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
30 lines
1.2 KiB
TypeScript
30 lines
1.2 KiB
TypeScript
import env from '#start/env'
|
||
import app from '@adonisjs/core/services/app'
|
||
|
||
/** The coordinator (REBUILD.md, part 1). */
|
||
export default {
|
||
/** Articles per range. */
|
||
rangeArticles: env.get('RANGE_ARTICLES', 2_000_000),
|
||
/**
|
||
* Articles per XOVER chunk (spool files, stream frames). Eweka takes
|
||
* ~1.5–2 s per command whatever its size: one connection read 9–15k
|
||
* lines/s in chunks of 25k, ~20k in 50k, ~30k in 100k and 200k (LOG.md
|
||
* 2026-10-09).
|
||
*/
|
||
chunkArticles: env.get('CHUNK_ARTICLES', 100_000),
|
||
/** A leased range without a heartbeat for this long goes back to the queue. */
|
||
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
|
||
/** NNTP connections the coordinator fetches headers with. */
|
||
nntpConnections: env.get('NNTP_CONNECTIONS', 4),
|
||
/** A reply with no data for this long fails (old ranges answer slowly). */
|
||
nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 10 * 60 * 1000),
|
||
spool: {
|
||
dir: env.get('SPOOL_DIR', app.tmpPath('spool')),
|
||
/** Ranges kept fetched ahead, per active lease. */
|
||
aheadPerLease: env.get('SPOOL_AHEAD_PER_LEASE', 2),
|
||
/** At least this many ranges ahead, so the first leases start at once. */
|
||
minAhead: env.get('SPOOL_MIN_AHEAD', 2),
|
||
maxBytes: env.get('SPOOL_MAX_GB', 10) * 1e9,
|
||
},
|
||
}
|