usenet-indexer/app/services/SeedFetchPool.ts

import nntpConfig from '#config/nntp'
import indexerConfig from '#config/indexer'
import { XoverClient } from '#lib/nntp/xover_client'

interface Slot {
  client: XoverClient
  group?: string
  usedAt: number
}

/**
 * A pooled connection unused this long is replaced: the news server drops
 * idle connections without telling, and a request on one then hangs until
 * the inactivity timeout (LOG.md 2026-10-05 (11)).
 */
const IDLE_MS = 60 * 1000

/**
 * NNTP connections in the seed API for relay nodes (CONCEPT.md D38): the
 * indexer fetches overview chunks and passes the gzip stream on unopened, so
 * nodes need no news server account and every connection comes from this
 * machine's IP. SEED_NNTP_CONNECTIONS (0: relay off).
 */
class SeedFetchPool {
  private idle: Slot[] = []
  private open = 0
  private waiters: (() => void)[] = []

  get size() {
    return indexerConfig.seed.nntpConnections
  }

  /**
   * Pooled connections fetch compressed only with SEED_NNTP_COMPRESS (plain is
   * faster for recent articles). A failure is retried once without compression, on a connection of its
   * own: either the pooled connection was dead (dropped while idle), or the
   * server stalled mid-reply in compressed mode. Eweka does that for some
   * chunks: it stops after most of the stream and never sends the rest
   * (a.b.ath 49701720000: 19,672 of 20,000 overviews, then silence), while
   * the same chunk uncompressed comes in 0.3 s (LOG.md 2026-10-05 (11)).
   */
  async fetch(group: string, from: number, to: number): Promise<Buffer> {
    try {
      return await this.fetchOnce(group, from, to)
    } catch {
      return this.fetchPlain(group, from, to)
    }
  }

  private async fetchPlain(group: string, from: number, to: number): Promise<Buffer> {
    const client = await this.connect(false)
    try {
      await client.group(group)
      return await client.xoverRaw(from, to)
    } finally {
      client.close()
    }
  }

  private connect(compress: boolean) {
    return XoverClient.connect({
      host: nntpConfig.host,
      port: nntpConfig.port,
      secure: nntpConfig.secure,
      servername: nntpConfig.tlsServername,
      user: nntpConfig.user,
      password: nntpConfig.password?.release(),
      timeoutMs: indexerConfig.seed.nntpTimeoutMs,
      compress,
    })
  }

  private async fetchOnce(group: string, from: number, to: number): Promise<Buffer> {
    const slot = await this.acquire()
    let ok = false
    try {
      if (slot.group !== group) {
        await slot.client.group(group)
        slot.group = group
      }
      const raw = await slot.client.xoverRaw(from, to)
      ok = true
      return raw
    } finally {
      this.release(slot, ok)
    }
  }

  /** Closes the idle connections (tests, shutdown); busy ones close when released. */
  closeIdle() {
    for (const slot of this.idle.splice(0)) {
      slot.client.close()
      this.open--
    }
  }

  private async acquire(): Promise<Slot> {
    while (true) {
      const slot = this.idle.pop()
      if (slot) {
        if (Date.now() - slot.usedAt < IDLE_MS) return slot
        slot.client.close()
        this.open--
        continue
      }
      if (this.open < this.size) {
        this.open++
        try {
          return { client: await this.connect(indexerConfig.seed.nntpCompress), usedAt: Date.now() }
        } catch (error) {
          this.open--
          this.waiters.shift()?.()
          throw error
        }
      }
      await new Promise<void>((resolve) => this.waiters.push(resolve))
    }
  }

  private release(slot: Slot, ok: boolean) {
    if (ok) {
      slot.usedAt = Date.now()
      this.idle.push(slot)
    } else {
      // A failed command can leave the connection mid-reply: start afresh.
      slot.client.close()
      this.open--
    }
    this.waiters.shift()?.()
  }
}

export default new SeedFetchPool()