Rebuild: hedged chunk fetches and connection max age (both off by default)

A chunk not in after NNTP_HEDGE_AFTER_SECONDS (10) is asked again on one
of NNTP_HEDGE_CONNECTIONS spare connections; the first answer wins and
the other connection is closed (lib/nntp/hedge.ts, tested). Connections
can be retired after NNTP_CONNECTION_MAX_MINUTES (±25%). The prefetch
report counts hedges and slow first requests by connection age.

Homelab, 15 min each: hedging on (40 + 6) 280k articles/s, off 273k.
The second request won about half of its races, but ~25-35% of first
requests were slow (>10 s) at every connection age, so neither helps:
Eweka is slow under our load, not on particular connections. Both
off by default; the age statistics stay.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/nntp_pool.ts

@@ -6,20 +6,38 @@ import { XoverClient } from '#lib/nntp/index'
 /**
  * The coordinator's header connections (REBUILD.md, decision e: workers
  * have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
- * that failed is closed and replaced on the next use.
+ * that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
+ * replaced on the next use.
  */
 class NntpPool {
   private idle: XoverClient[] = []
   private busy = new Set<XoverClient>()
   private open = 0
   private waiting: ((client: XoverClient) => void)[] = []
+  private openedAt = new Map<XoverClient, number>()
+  /** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
+  private retireAt = new Map<XoverClient, number>()
 
   get size() {
     return coordinatorConfig.nntpConnections
   }
 
   async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
-    const client = await this.acquire()
+    return this.run(await this.acquire(), work)
+  }
+
+  /** As use(), if a connection is free or can be opened now; else null, without waiting. */
+  tryUse<T>(work: (client: XoverClient) => Promise<T>): Promise<T> | null {
+    if (this.idle.length === 0 && this.open >= this.size) return null
+    return this.acquire().then((client) => this.run(client, work))
+  }
+
+  /** How long ago the connection was opened, in ms. */
+  ageOf(client: XoverClient): number {
+    return Date.now() - (this.openedAt.get(client) ?? Date.now())
+  }
+
+  private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
     this.busy.add(client)
     let broken = false
     try {
@@ -39,7 +57,7 @@ class NntpPool {
     if (this.open < this.size) {
       this.open++
       try {
-        return await XoverClient.connect({
+        const client = await XoverClient.connect({
           host: env.get('NNTP_HOST')!,
           port: env.get('NNTP_PORT', 563),
           user: env.get('NNTP_USER'),
@@ -48,6 +66,10 @@ class NntpPool {
           compress: true,
           timeoutMs: coordinatorConfig.nntpTimeoutMs,
         })
+        this.openedAt.set(client, Date.now())
+        const maxMs = coordinatorConfig.nntpConnectionMaxMs
+        if (maxMs > 0) this.retireAt.set(client, Date.now() + maxMs * (0.75 + Math.random() / 2))
+        return client
       } catch (error) {
         this.open--
         throw error
@@ -57,8 +79,10 @@ class NntpPool {
   }
 
   private release(client: XoverClient, broken: boolean) {
-    if (broken) {
+    if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
       this.open--
+      this.openedAt.delete(client)
+      this.retireAt.delete(client)
       try {
         client.close()
       } catch (error: any) {
@@ -112,6 +136,8 @@ class NntpPool {
     this.plain = null
     for (const client of [...this.idle.splice(0), ...this.busy]) client.close()
     this.busy.clear()
+    this.openedAt.clear()
+    this.retireAt.clear()
     this.open = 0
   }
 }