Daan MeijerandClaude Opus 5.5
530a752cca
- A range reported to its end but not finished (its node went away at the very end) was leased again with start past end; nodes asked for "from 49208000000 to 49207999999", got 400, and the range failed after five leases. The orchestrator now marks such ranges done before leasing, and runRange reports them done without fetching. - Some old ranges come so slowly that one XOVER of 20k headers takes over two minutes: five ranges failed on "NNTP reply timed out". The relay pool's timeout is SEED_NNTP_TIMEOUT_MS (default 10 minutes, the node's HTTP timeout). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
92 lines
2.4 KiB
TypeScript
92 lines
2.4 KiB
TypeScript
import nntpConfig from '#config/nntp'
|
|
import indexerConfig from '#config/indexer'
|
|
import { XoverClient } from '#lib/nntp/xover_client'
|
|
|
|
interface Slot {
|
|
client: XoverClient
|
|
group?: string
|
|
usedAt: number
|
|
}
|
|
|
|
/** A pooled connection unused this long is replaced: the news server may have dropped it. */
|
|
const IDLE_MS = 2 * 60 * 1000
|
|
|
|
/**
|
|
* NNTP connections in the seed API for relay nodes (CONCEPT.md D38): the
|
|
* indexer fetches overview chunks and passes the gzip stream on unopened, so
|
|
* nodes need no news server account and every connection comes from this
|
|
* machine's IP. SEED_NNTP_CONNECTIONS (0: relay off).
|
|
*/
|
|
class SeedFetchPool {
|
|
private idle: Slot[] = []
|
|
private open = 0
|
|
private waiters: (() => void)[] = []
|
|
|
|
get size() {
|
|
return indexerConfig.seed.nntpConnections
|
|
}
|
|
|
|
async fetch(group: string, from: number, to: number): Promise<Buffer> {
|
|
const slot = await this.acquire()
|
|
let ok = false
|
|
try {
|
|
if (slot.group !== group) {
|
|
await slot.client.group(group)
|
|
slot.group = group
|
|
}
|
|
const raw = await slot.client.xoverRaw(from, to)
|
|
ok = true
|
|
return raw
|
|
} finally {
|
|
this.release(slot, ok)
|
|
}
|
|
}
|
|
|
|
private async acquire(): Promise<Slot> {
|
|
while (true) {
|
|
const slot = this.idle.pop()
|
|
if (slot) {
|
|
if (Date.now() - slot.usedAt < IDLE_MS) return slot
|
|
slot.client.close()
|
|
this.open--
|
|
continue
|
|
}
|
|
if (this.open < this.size) {
|
|
this.open++
|
|
try {
|
|
const client = await XoverClient.connect({
|
|
host: nntpConfig.host,
|
|
port: nntpConfig.port,
|
|
secure: nntpConfig.secure,
|
|
servername: nntpConfig.tlsServername,
|
|
user: nntpConfig.user,
|
|
password: nntpConfig.password?.release(),
|
|
timeoutMs: indexerConfig.seed.nntpTimeoutMs,
|
|
compress: true,
|
|
})
|
|
return { client, usedAt: Date.now() }
|
|
} catch (error) {
|
|
this.open--
|
|
this.waiters.shift()?.()
|
|
throw error
|
|
}
|
|
}
|
|
await new Promise<void>((resolve) => this.waiters.push(resolve))
|
|
}
|
|
}
|
|
|
|
private release(slot: Slot, ok: boolean) {
|
|
if (ok) {
|
|
slot.usedAt = Date.now()
|
|
this.idle.push(slot)
|
|
} else {
|
|
// A failed command can leave the connection mid-reply: start afresh.
|
|
slot.client.close()
|
|
this.open--
|
|
}
|
|
this.waiters.shift()?.()
|
|
}
|
|
}
|
|
|
|
export default new SeedFetchPool()
|