From 887df12ead603cd0a2da0ce7d4424f6dac366d17 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sat, 10 Oct 2026 16:51:55 +0200 Subject: [PATCH] Rebuild: give up stalled requests, not slow ones a.b.0day.stuffz sends steadily but slowly (~400 kB a minute, 5-7 minutes per 100k chunk): the fixed deadline (90 s, doubling up to 360 s) gave its chunks up again and again, and two of its ranges' worker streams timed out. A stall is a request that got less than NNTP_STALL_KB_PER_MINUTE (100) in the last minute (the stuck replies got ~25 kB a minute); then its connection is closed and the chunk fetched again. Replaces NNTP_CHUNK_DEADLINE_SECONDS. Co-Authored-By: Claude Opus 5.5 --- app/services/prefetcher.ts | 35 ++++++++++++++++++++--------------- config/coordinator.ts | 16 ++++++++-------- start/env.ts | 2 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/app/services/prefetcher.ts b/app/services/prefetcher.ts index 8734da6..ff0d485 100644 --- a/app/services/prefetcher.ts +++ b/app/services/prefetcher.ts @@ -16,6 +16,10 @@ interface Target { total: number } +/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */ +const STALL_SAMPLE_MS = 15_000 +const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS + /** Connection ages the report counts slow first requests by, in minutes. */ const AGE_BUCKETS = [2, 10, Infinity] @@ -59,8 +63,6 @@ export class Prefetcher { private fetchingNow = 0 private writingNow = 0 private sampler?: NodeJS.Timeout - /** Deadlines a chunk hit, by range and chunk start: each doubles its next deadline, so a slow chunk still comes. */ - private lateChunks = new Map() /** Requests in flight: what, since when, on which connection (the report names the oldest). */ private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>() @@ -88,7 +90,7 @@ export class Prefetcher { idle: 0, inFlight: 0, inFlightMin: Infinity, - /** Requests given up at nntpChunkDeadlineMs. */ + /** Requests given up as stalled (nntpStallBytesPerMinute). */ deadlines: 0, ranges: 0, groups: 0, @@ -191,15 +193,19 @@ export class Prefetcher { if (first) age = NntpPool.ageOf(c) const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c } this.inFlight.add(entry) + // Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall. let late = false - const key = `${rangeId}:${span.from}` - const deadlineMs = coordinatorConfig.nntpChunkDeadlineMs * 2 ** Math.min(this.lateChunks.get(key) ?? 0, 4) - const deadline = setTimeout(() => { - late = true - this.stats.deadlines++ - this.lateChunks.set(key, (this.lateChunks.get(key) ?? 0) + 1) - c.close() - }, deadlineMs) + const samples: number[] = [] + const watch = setInterval(() => { + samples.push(c.buffered) + if (samples.length <= STALL_SAMPLES) return + const minuteAgo = samples.shift()! + if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) { + late = true + this.stats.deadlines++ + c.close() + } + }, STALL_SAMPLE_MS) let data: Buffer let groupMs = 0 try { @@ -209,12 +215,11 @@ export class Prefetcher { data = await c.xoverRaw(span.from, span.to) } catch (error) { if (!late) throw error - throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} not in after ${deadlineMs / 1000} s; fetching it again`) + throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`) } finally { - clearTimeout(deadline) + clearInterval(watch) this.inFlight.delete(entry) } - this.lateChunks.delete(key) if (!cancelled) { const t = c.lastTiming this.stats.requests++ @@ -294,7 +299,7 @@ export class Prefetcher { `loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` + `a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` + `${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` + - `${st.deadlines} given up after ${coordinatorConfig.nntpChunkDeadlineMs / 1000} s; ${this.oldest()}.` + `${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.` ) } this.stats = this.freshStats() diff --git a/config/coordinator.ts b/config/coordinator.ts index f8c24b4..03ee2e0 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -38,15 +38,15 @@ export default { */ nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 2 * 60 * 1000), /** - * A chunk still coming in after this long is given up (its connection - * closed) and fetched again: a reply that trickles never goes quiet long - * enough for nntpTimeoutMs, and kept a worker waiting on one chunk for an - * hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s; at 300 s, 258 were - * given up in 3 hours (~1.4 a minute, each holding a connection 5 minutes); - * at 180 s 60 in 15 minutes, ~30% of 40 connections' time, while finished - * chunks took 24 s on average. + * A request that got less than this in the last minute (kB) is given up, + * its connection closed, and the chunk fetched again: a reply that trickles + * (6,204 → 6,229 kB in a minute) never goes quiet long enough for + * nntpTimeoutMs, and kept a worker waiting on one chunk for an hour. Not a + * deadline: some groups send steadily but slowly (a.b.0day.stuffz ~400 kB a + * minute, 5–7 minutes a chunk), and a fixed 90–360 s gave those up again + * and again (LOG.md 2026-10-10 (3)). */ - nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 90) * 1000, + nntpStallBytesPerMinute: env.get('NNTP_STALL_KB_PER_MINUTE', 100) * 1000, /** * A chunk not fetched after this long is asked again on another * connection; the first answer wins and the other connection is closed. diff --git a/start/env.ts b/start/env.ts index 671e2d2..cd83f7e 100644 --- a/start/env.ts +++ b/start/env.ts @@ -47,7 +47,7 @@ export default await Env.create(new URL('../', import.meta.url), { NNTP_HEDGE_AFTER_SECONDS: Env.schema.number.optional(), NNTP_HEDGE_CONNECTIONS: Env.schema.number.optional(), NNTP_CONNECTION_MAX_MINUTES: Env.schema.number.optional(), - NNTP_CHUNK_DEADLINE_SECONDS: Env.schema.number.optional(), + NNTP_STALL_KB_PER_MINUTE: Env.schema.number.optional(), // Newznab API key (empty: no key needed) NEWZNAB_API_KEY: Env.schema.secret.optional(),