Seeding nodes log in before leasing; TLS through tunnels

- seed:node logs in to NNTP before taking a range and keeps the connection across ranges (replaced after 2 minutes idle or a failed range). A node that can't reach its news server takes no range: two ranges were set aside as failed while an office node's tunnel was being set up (ECONNREFUSED 127.0.0.1:1563, five leases each). After an IP-address limit refusal it waits 5 minutes. - NNTP_TLS_SERVERNAME: the hostname the TLS certificate is checked against when NNTP_HOST is a tunnel ("Hostname/IP does not match certificate's altnames: Host: localhost"). - seed-node/README: the IP-address limit and tunnelling. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 14:14:15 +02:00
1 parent 04f945055f
commit 70036847cf
@@ -29,7 +29,8 @@ interface Lease {
 */
 export default class SeedNode extends BaseCommand {
  public static commandName = 'seed:node'
-  public static description = 'Runs a seeding node: leases ranges from the seed API and submits what it aggregated.'
+  public static description =
+    'Runs a seeding node: leases ranges from the seed API and submits what it aggregated.'

  public static options: CommandOptions = {
    startApp: false,
@@ -42,7 +43,11 @@ export default class SeedNode extends BaseCommand {
  @flags.string({ description: 'Node token (default: SEED_TOKEN)' })
  declare token?: string

-  @flags.number({ description: 'Ranges worked on at once, one NNTP connection each (default: SEED_CONNECTIONS or 1)', default: Number(process.env.SEED_CONNECTIONS) || 1 })
+  @flags.number({
+    description:
+      'Ranges worked on at once, one NNTP connection each (default: SEED_CONNECTIONS or 1)',
+    default: Number(process.env.SEED_CONNECTIONS) || 1,
+  })
  declare connections: number

  @flags.number({ description: 'Articles per XOVER', default: 20_000 })
@@ -79,83 +84,130 @@ export default class SeedNode extends BaseCommand {

  private async work(api: SeedApi, worker: number) {
    const log = (message: string) => this.logger.info(`[${worker}] ${message}`)
-    while (!this.stopping) {
-      let lease: Lease | null
-      try {
-        lease = ((await api.post('/api/v1/seed/lease')) as { range: Lease } | null)?.range ?? null
-      } catch (error: any) {
-        if (error instanceof Unauthorized) throw error
-        log(`Lease failed: ${error.message}`)
-        await this.pause(60_000)
-        continue
-      }
-      if (!lease) {
-        if (this.once) return
-        await this.pause(60_000)
-        continue
-      }
-
-      log(`${lease.group} ${lease.start}–${lease.end}`)
-      const started = Date.now()
-      let headers = 0
-      let client: XoverClient | undefined
-      let chain: Promise<void> = Promise.resolve()
-      const posting = new Set<Promise<void>>()
-      // Throws once a post has failed, so the range stops fetching.
-      const failedSoFar = () => Promise.race([chain, Promise.resolve()])
-      try {
-        client = await XoverClient.connect({
-          host: nntpConfig.host,
-          port: nntpConfig.port,
-          secure: nntpConfig.secure,
-          user: nntpConfig.user,
-          password: nntpConfig.password?.release(),
-          timeoutMs: nntpConfig.commandTimeoutMs,
-          compress: true,
-        })
-        await runRange({
-          client,
-          group: lease.group,
-          start: lease.start,
-          end: lease.end,
-          chunk: this.chunk,
-          submitEvery: this.submitEvery,
-          isDeferred: (header: OverviewHeader) => indexerConfig.deferBodyMessageIds.some((re) => re.test(header['message-id'] ?? '')),
-          // Batches are posted in order, one after the other, while fetching goes on; with two
-          // waiting, fetching waits too. A failed post stops the chain, so no batch is stored
-          // past a missing one (the range would resume beyond it).
-          sink: async (batch: SeedBatch) => {
-            if (this.stopping) throw new Stopping()
-            while (posting.size >= 2) await Promise.race(posting)
-            const post = chain.then(async () => {
-              await api.post(`/api/v1/seed/ranges/${lease!.id}/batches`, batch)
-              headers += batch.headers
-            })
-            chain = post
-            posting.add(post)
-            post.catch(() => {}).finally(() => posting.delete(post))
-            await failedSoFar()
-          },
-        })
-        await chain
-        await api.post(`/api/v1/seed/ranges/${lease.id}/done`)
-        const seconds = (Date.now() - started) / 1000
-        log(`done: ${headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s (${Math.round(headers / seconds).toLocaleString('en')}/s)`)
-      } catch (error: any) {
-        if (error instanceof Unauthorized) throw error
-        if (error instanceof LeaseLost) {
-          log(`Range ${lease.id} passed to another node; dropping it.`)
-          continue
+    // Logged in before leasing, so a node that can't reach its news server takes no range
+    // (each lease counts as an attempt; a range is set aside after five). Kept across ranges.
+    let client: XoverClient | undefined
+    let idleSince = Date.now()
+    try {
+      while (!this.stopping) {
+        // A news server may drop a connection that sat idle (nothing to lease): log in afresh.
+        if (client && Date.now() - idleSince > 2 * 60_000) {
+          client.close()
+          client = undefined
        }
-        const reason = error instanceof Stopping ? 'node stopping' : error.message
-        if (!(error instanceof Stopping)) log(`Range ${lease.id} failed: ${reason}`)
-        // Let posts in flight finish (or fail) first, so the range resumes after what was stored.
-        await chain.catch(() => {})
-        await api.post(`/api/v1/seed/ranges/${lease.id}/release`, { error: reason }).catch(() => {})
-        if (!this.stopping) await this.pause(30_000)
-      } finally {
-        client?.close()
+        if (!client) {
+          try {
+            client = await this.connect()
+          } catch (error: any) {
+            const ipLimit = /simultaneous IP/i.test(error.message)
+            log(`NNTP login failed: ${error.message}; trying again in ${ipLimit ? 5 : 1} min`)
+            await this.pause(ipLimit ? 300_000 : 60_000)
+            continue
+          }
+        }
+        const outcome = await this.range(api, client, log)
+        if (outcome === 'replace') {
+          client.close()
+          client = undefined
+        }
+        if (outcome !== 'idle') idleSince = Date.now()
      }
+    } finally {
+      client?.close()
+    }
+  }
+
+  private connect() {
+    return XoverClient.connect({
+      host: nntpConfig.host,
+      port: nntpConfig.port,
+      secure: nntpConfig.secure,
+      servername: nntpConfig.tlsServername,
+      user: nntpConfig.user,
+      password: nntpConfig.password?.release(),
+      timeoutMs: nntpConfig.commandTimeoutMs,
+      compress: true,
+    })
+  }
+
+  /**
+   * Leases and works one range: 'done', 'idle' (nothing leased; the connection
+   * wasn't used), or 'replace' (the connection may be in an odd state).
+   */
+  private async range(
+    api: SeedApi,
+    client: XoverClient,
+    log: (message: string) => void
+  ): Promise<'done' | 'idle' | 'replace'> {
+    let lease: Lease | null
+    try {
+      lease = ((await api.post('/api/v1/seed/lease')) as { range: Lease } | null)?.range ?? null
+    } catch (error: any) {
+      if (error instanceof Unauthorized) throw error
+      log(`Lease failed: ${error.message}`)
+      await this.pause(60_000)
+      return 'idle'
+    }
+    if (!lease) {
+      if (this.once) this.stopping = true
+      else await this.pause(60_000)
+      return 'idle'
+    }
+
+    log(`${lease.group} ${lease.start}–${lease.end}`)
+    const started = Date.now()
+    let headers = 0
+    let chain: Promise<void> = Promise.resolve()
+    const posting = new Set<Promise<void>>()
+    // Throws once a post has failed, so the range stops fetching.
+    const failedSoFar = () => Promise.race([chain, Promise.resolve()])
+    try {
+      await runRange({
+        client,
+        group: lease.group,
+        start: lease.start,
+        end: lease.end,
+        chunk: this.chunk,
+        submitEvery: this.submitEvery,
+        isDeferred: (header: OverviewHeader) =>
+          indexerConfig.deferBodyMessageIds.some((re) => re.test(header['message-id'] ?? '')),
+        // Batches are posted in order, one after the other, while fetching goes on; with two
+        // waiting, fetching waits too. A failed post stops the chain, so no batch is stored
+        // past a missing one (the range would resume beyond it).
+        sink: async (batch: SeedBatch) => {
+          if (this.stopping) throw new Stopping()
+          while (posting.size >= 2) await Promise.race(posting)
+          const post = chain.then(async () => {
+            await api.post(`/api/v1/seed/ranges/${lease!.id}/batches`, batch)
+            headers += batch.headers
+          })
+          chain = post
+          posting.add(post)
+          post.catch(() => {}).finally(() => posting.delete(post))
+          await failedSoFar()
+        },
+      })
+      await chain
+      await api.post(`/api/v1/seed/ranges/${lease.id}/done`)
+      const seconds = (Date.now() - started) / 1000
+      log(
+        `done: ${headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s (${Math.round(headers / seconds).toLocaleString('en')}/s)`
+      )
+      return 'done'
+    } catch (error: any) {
+      if (error instanceof Unauthorized) throw error
+      // An XOVER may still be in flight: the connection is replaced either way.
+      if (error instanceof LeaseLost) {
+        log(`Range ${lease.id} passed to another node; dropping it.`)
+        return 'replace'
+      }
+      const reason = error instanceof Stopping ? 'node stopping' : error.message
+      if (!(error instanceof Stopping)) log(`Range ${lease.id} failed: ${reason}`)
+      // Let posts in flight finish (or fail) first, so the range resumes after what was stored.
+      await chain.catch(() => {})
+      await api.post(`/api/v1/seed/ranges/${lease.id}/release`, { error: reason }).catch(() => {})
+      if (!this.stopping) await this.pause(30_000)
+      return 'replace'
    }
  }

@@ -184,8 +236,8 @@ class SeedApi {
        const response = await fetch(this.base + path, {
          method: 'POST',
          headers: {
-            'authorization': `Bearer ${this.token}`,
-            'accept': 'application/json',
+            authorization: `Bearer ${this.token}`,
+            accept: 'application/json',
            ...(payload ? { 'content-type': 'application/json', 'content-encoding': 'gzip' } : {}),
          },
          body: payload,
@@ -201,9 +253,11 @@ class SeedApi {
      if (status === 401) throw new Unauthorized('The seed API rejected this node token')
      if (status === 409) throw new LeaseLost(text)
      // 404 comes from the proxy while the seed API restarts (a deploy): this API has no 404s of its own.
-      if (status >= 400 && status < 500 && status !== 404) throw new Error(`Seed API ${path}: ${status} ${text.slice(0, 300)}`)
+      if (status >= 400 && status < 500 && status !== 404)
+        throw new Error(`Seed API ${path}: ${status} ${text.slice(0, 300)}`)
      // Network errors, 404 and 5xx: storing a batch twice is harmless, so try again.
-      if (attempt >= 8) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text.slice(0, 300)}`)
+      if (attempt >= 8)
+        throw new Error(`Seed API ${path}: ${status || 'no response'} ${text.slice(0, 300)}`)
      const wait = Math.min(120_000, 2000 * 2 ** (attempt - 1))
      this.warn(`Seed API ${path}: ${status || text}; retrying in ${wait / 1000} s`)
      await sleep(wait)
@@ -60,6 +60,7 @@ export default class SeedRange extends BaseCommand {
      host: nntpConfig.host,
      port: nntpConfig.port,
      secure: nntpConfig.secure,
+      servername: nntpConfig.tlsServername,
      user: nntpConfig.user,
      password: nntpConfig.password?.release(),
      timeoutMs: nntpConfig.commandTimeoutMs,