From 3e6b5bed569bc5124eeb47f0b687fa9a12e89bd3 Mon Sep 17 00:00:00 2001
From: Daan Meijer <daan@studioseptember.nl>
Date: Tue, 6 Oct 2026 15:49:21 +0200
Subject: [PATCH] Seed relay fetches plain overviews by default

For 2026 a.b.ath articles Eweka's gzip replies have a long tail (p90 12.4 s
per 20k vs 3.9 s plain; 6.1k vs 10.5k headers/s). SEED_NNTP_COMPRESS turns
compression back on; plain replies are deflated for nodes asynchronously.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 app/services/SeedFetchPool.ts      | 5 +++--
 config/indexer.ts                  | 6 ++++++
 lib/nntp/xover_client.ts           | 9 +++++++--
 start/env.ts                       | 1 +
 tests/unit/seed_fetch_pool.spec.ts | 5 ++++-
 5 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/app/services/SeedFetchPool.ts b/app/services/SeedFetchPool.ts
index 9fbee76..782dbe9 100644
--- a/app/services/SeedFetchPool.ts
+++ b/app/services/SeedFetchPool.ts
@@ -31,7 +31,8 @@ class SeedFetchPool {
   }
 
   /**
-   * A failure is retried once without compression, on a connection of its
+   * Pooled connections fetch compressed only with SEED_NNTP_COMPRESS (plain is
+   * faster for recent articles). A failure is retried once without compression, on a connection of its
    * own: either the pooled connection was dead (dropped while idle), or the
    * server stalled mid-reply in compressed mode. Eweka does that for some
    * chunks: it stops after most of the stream and never sends the rest
@@ -105,7 +106,7 @@ class SeedFetchPool {
       if (this.open < this.size) {
         this.open++
         try {
-          return { client: await this.connect(true), usedAt: Date.now() }
+          return { client: await this.connect(indexerConfig.seed.nntpCompress), usedAt: Date.now() }
         } catch (error) {
           this.open--
           this.waiters.shift()?.()
diff --git a/config/indexer.ts b/config/indexer.ts
index e5d5e17..117d0e9 100644
--- a/config/indexer.ts
+++ b/config/indexer.ts
@@ -72,6 +72,12 @@ export default {
      * answered" in 10 minutes came in 0.3 s on a fresh connection.
      */
     nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 90 * 1000),
+    /**
+     * Ask the news server for gzip-compressed overviews. Off: for 2026 articles Eweka's
+     * compressed replies have a long tail (p90 12.4 s per 20k, against 3.9 s plain; 6.1k
+     * against 10.5k headers/s, LOG.md 2026-10-06 (9)). Nodes get gzip from the relay either way.
+     */
+    nntpCompress: env.get('SEED_NNTP_COMPRESS', false),
   },
 
   backfill: {
diff --git a/lib/nntp/xover_client.ts b/lib/nntp/xover_client.ts
index 67033bc..e592cee 100644
--- a/lib/nntp/xover_client.ts
+++ b/lib/nntp/xover_client.ts
@@ -1,6 +1,9 @@
+import { promisify } from 'node:util'
 import net from 'node:net'
 import tls from 'node:tls'
-import { deflateSync, unzipSync } from 'node:zlib'
+import { deflate, deflateSync, unzipSync } from 'node:zlib'
+
+const deflateAsync = promisify(deflate)
 
 export interface XoverClientOptions {
   host: string
@@ -100,7 +103,9 @@ export class XoverClient {
       const reply = await this.command(`XOVER ${from}-${to}`)
       if (reply.startsWith('420') || reply.startsWith('423')) return deflateSync(Buffer.alloc(0))
       await this.expect(reply, ['224'])
-      return reply.includes('COMPRESS=GZIP') ? (await this.compressedRaw()).raw : deflateSync(Buffer.from(await this.plainBody()))
+      if (reply.includes('COMPRESS=GZIP')) return (await this.compressedRaw()).raw
+      // Plain replies (the relay's default): compressed off the event loop, at the fastest level.
+      return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
     })
   }
 
diff --git a/start/env.ts b/start/env.ts
index a2b5a12..5ebf0f7 100644
--- a/start/env.ts
+++ b/start/env.ts
@@ -83,6 +83,7 @@ export default await Env.create(new URL('../', import.meta.url), {
   NNTP_TLS_SERVERNAME: Env.schema.string.optional(),
   SEED_NNTP_CONNECTIONS: Env.schema.number.optional(),
   SEED_NNTP_TIMEOUT_MS: Env.schema.number.optional(),
+  SEED_NNTP_COMPRESS: Env.schema.boolean.optional(),
 
   // Indexer (config/indexer.ts)
   BACKFILL_MAX_QUEUED: Env.schema.number.optional(),
diff --git a/tests/unit/seed_fetch_pool.spec.ts b/tests/unit/seed_fetch_pool.spec.ts
index bbe87ab..ccd31c9 100644
--- a/tests/unit/seed_fetch_pool.spec.ts
+++ b/tests/unit/seed_fetch_pool.spec.ts
@@ -26,7 +26,9 @@ test.group('SeedFetchPool', () => {
       })
     })
     await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
-    const saved = { host: nntpConfig.host, port: nntpConfig.port, secure: nntpConfig.secure, connections: indexerConfig.seed.nntpConnections }
+    const saved = { host: nntpConfig.host, port: nntpConfig.port, secure: nntpConfig.secure, connections: indexerConfig.seed.nntpConnections, compress: indexerConfig.seed.nntpCompress }
+    // The fallback from a compressed pooled connection (SEED_NNTP_COMPRESS on).
+    indexerConfig.seed.nntpCompress = true
     Object.assign(nntpConfig, { host: '127.0.0.1', port: (server.address() as net.AddressInfo).port, secure: false })
     indexerConfig.seed.nntpConnections = 2
     try {
@@ -39,6 +41,7 @@ test.group('SeedFetchPool', () => {
       SeedFetchPool.closeIdle()
       Object.assign(nntpConfig, { host: saved.host, port: saved.port, secure: saved.secure })
       indexerConfig.seed.nntpConnections = saved.connections
+      indexerConfig.seed.nntpCompress = saved.compress
       server.close()
     }
   })