usenet-indexer/tests/unit/nntp/xover_client.spec.ts

import { test } from '@japa/runner'
import net from 'node:net'
import { deflateSync } from 'node:zlib'
import { MissingArticle, TruncatedReply, XoverClient, parseOverviews } from '#lib/nntp/index'
import { unzipSync } from 'node:zlib'

const LINES = [
  '101\tShow.S01E01 - "a.mkv" yEnc (1/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<one@test>\t\t700000\t5000\tXref: news.example a.b.test:101',
  '102\tShow.S01E01 - "a.mkv" yEnc (2/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:01 +0000\t<two@test>\t\t650000\t4600\tXref: news.example a.b.test:102 a.b.other:7',
]

/** A tiny NNTP server: plain replies, gzip after XFEATURE, 420 for 1-1, replies split into small chunks. */
function server(): Promise<{ port: number; close: () => void }> {
  return new Promise((resolve) => {
    const srv = net.createServer((sock) => {
      let gzip = false
      const send = (data: Buffer) => {
        for (let i = 0; i < data.length; i += 7) sock.write(data.subarray(i, i + 7))
      }
      sock.write('200 welcome\r\n')
      let pending = ''
      sock.on('data', (chunk) => {
        pending += chunk.toString()
        let at: number
        while ((at = pending.indexOf('\r\n')) !== -1) {
          const line = pending.slice(0, at)
          pending = pending.slice(at + 2)
          if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
          else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
          else if (line === 'XFEATURE COMPRESS GZIP')
            ((gzip = true), sock.write('290 Feature Enabled\r\n'))
          else if (line.startsWith('GROUP')) sock.write('211 2 101 102 a.b.test\r\n')
          else if (line === 'XOVER 1-1') sock.write('420 No Articles Selected\r\n')
          else if (line === 'XOVER 5-5' && gzip) {
            // As Eweka cuts a compressed reply whose output has ".\r\n" early: the zlib header, then nothing.
            sock.write('224 Overview Information Follows [COMPRESS=GZIP]\r\n')
            sock.write(Buffer.from([0x78, 0x01, 0x2e, 0x0d, 0x0a]))
          } else if (line.startsWith('XOVER')) {
            const body = Buffer.from(LINES.join('\r\n') + '\r\n')
            if (gzip) {
              sock.write('224 Overview Information Follows [COMPRESS=GZIP]\r\n')
              send(Buffer.concat([deflateSync(body), Buffer.from('.\r\n')]))
            } else {
              sock.write('224 Overview Information Follows\r\n')
              send(Buffer.concat([body, Buffer.from('.\r\n')]))
            }
          } else if (line.startsWith('LIST ACTIVE')) {
            // As Eweka: compressed too once XFEATURE COMPRESS GZIP is on.
            const body = Buffer.from('a.b.test 102 101 y\r\na.b.other 9000000000 2 y\r\n')
            if (gzip) {
              sock.write('215 NewsGroups Follow [COMPRESS=GZIP]\r\n')
              send(Buffer.concat([deflateSync(body), Buffer.from('.\r\n')]))
            } else {
              sock.write('215 NewsGroups Follow\r\n')
              send(Buffer.concat([body, Buffer.from('.\r\n')]))
            }
          } else if (line === 'BODY <gone@test>') sock.write('430 No Such Article\r\n')
          else if (line.startsWith('BODY')) {
            sock.write('222 0 <one@test> body\r\n')
            // Raw bytes, a dot-stuffed line among them.
            send(
              Buffer.concat([
                Buffer.from([0x3d, 0x79, 0xff, 0x00, 0x0d, 0x0a]),
                Buffer.from('..stuffed\r\nlast\r\n.\r\n'),
              ])
            )
          } else if (line === 'QUIT') sock.end('205 bye\r\n')
        }
      })
    })
    srv.listen(0, '127.0.0.1', () =>
      resolve({ port: (srv.address() as net.AddressInfo).port, close: () => srv.close() })
    )
  })
}

test.group('nntp / XoverClient', () => {
  test('reads a body by message-id as bytes, raw; 430 is MissingArticle', async ({ assert }) => {
    const { port, close } = await server()
    const client = await XoverClient.connect({
      host: '127.0.0.1',
      port,
      user: 'u',
      password: 'p',
      compress: false,
    })
    try {
      const body = await client.body('one@test')
      assert.deepEqual([...body.subarray(0, 6)], [0x3d, 0x79, 0xff, 0x00, 0x0d, 0x0a])
      assert.equal(body.subarray(6).toString('latin1'), '..stuffed\r\nlast\r\n')
      await assert.rejects(() => client.body('<gone@test>'), MissingArticle)
      // The connection is still usable after both.
      assert.equal((await client.body('<one@test>')).length, body.length)
    } finally {
      client.close()
      close()
    }
  })

  test('reads several bodies pipelined, in order, a missing one as null', async ({ assert }) => {
    const { port, close } = await server()
    const client = await XoverClient.connect({
      host: '127.0.0.1',
      port,
      user: 'u',
      password: 'p',
      compress: false,
    })
    try {
      const bodies = await client.bodies(['one@test', '<gone@test>', '<one@test>'])
      assert.equal(bodies.length, 3)
      assert.equal(bodies[0]!.subarray(6).toString('latin1'), '..stuffed\r\nlast\r\n')
      assert.isNull(bodies[1])
      assert.equal(bodies[2]!.length, bodies[0]!.length)
      // The connection is still in step afterwards.
      assert.equal((await client.body('<one@test>')).length, bodies[0]!.length)
    } finally {
      client.close()
      close()
    }
  })

  test('parses overview lines into the fields nntp-js gives', ({ assert }) => {
    const [[number, fields]] = parseOverviews(LINES[0] + '\r\n')
    assert.equal(number, 101)
    assert.equal(fields['message-id'], '<one@test>')
    assert.equal(fields[':bytes'], '700000')
    assert.equal(fields.xref, 'news.example a.b.test:101')
  })

  for (const compress of [false, true]) {
    test(`fetches overviews ${compress ? 'gzip-compressed' : 'plain'}, and an empty range`, async ({
      assert,
    }) => {
      const { port, close } = await server()
      const client = await XoverClient.connect({
        host: '127.0.0.1',
        port,
        user: 'u',
        password: 'p',
        compress,
      })
      try {
        assert.equal(client.compressed, compress)
        assert.deepEqual(await client.group('a.b.test'), { count: 2, low: 101, high: 102 })
        const overviews = await client.xover(101, 102)
        assert.deepEqual(
          overviews.map(([n]) => n),
          [101, 102]
        )
        assert.equal(overviews[1][1].subject, 'Show.S01E01 - "a.mkv" yEnc (2/2)')
        assert.deepEqual(await client.xover(1, 1), [])
        assert.lengthOf(await client.xover(101, 102), 2)
        assert.deepEqual(await client.listActive('a.b.*'), [
          { name: 'a.b.test', high: 102, low: 101 },
          { name: 'a.b.other', high: 9_000_000_000, low: 2 },
        ])
      } finally {
        client.close()
        close()
      }
    })
  }

  test('a compressed reply cut short at a ".\\r\\n" in its output fails as TruncatedReply, well before the reply timeout', async ({
    assert,
  }) => {
    const { port, close } = await server()
    const client = await XoverClient.connect({
      host: '127.0.0.1',
      port,
      user: 'u',
      password: 'p',
      compress: true,
      timeoutMs: 60_000,
    })
    try {
      await client.group('a.b.test')
      const startedAt = Date.now()
      await assert.rejects(() => client.xover(5, 5), TruncatedReply)
      assert.isBelow(Date.now() - startedAt, 10_000)
      // The connection is still usable: the server thinks it answered.
      assert.lengthOf(await client.xover(101, 102), 2)
    } finally {
      client.close()
      close()
    }
  }).timeout(10_000)

  test('idle time between commands does not count against the reply timeout', async ({
    assert,
  }) => {
    const { port, close } = await server()
    const client = await XoverClient.connect({
      host: '127.0.0.1',
      port,
      user: 'u',
      password: 'p',
      compress: true,
      timeoutMs: 200,
    })
    try {
      await client.group('a.b.test')
      await new Promise((resolve) => setTimeout(resolve, 500))
      assert.lengthOf(await client.xover(101, 102), 2)
    } finally {
      client.close()
      close()
    }
  })

  for (const compress of [false, true]) {
    test(`hands out overviews as one zlib stream (${compress ? 'as the server sent it' : 'deflated here'})`, async ({
      assert,
    }) => {
      const { port, close } = await server()
      const client = await XoverClient.connect({
        host: '127.0.0.1',
        port,
        user: 'u',
        password: 'p',
        compress,
      })
      try {
        await client.group('a.b.test')
        const raw = await client.xoverRaw(101, 102)
        assert.deepEqual(
          parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n),
          [101, 102]
        )
        assert.lengthOf(parseOverviews(unzipSync(await client.xoverRaw(1, 1)).toString('utf8')), 0)
      } finally {
        client.close()
        close()
      }
    })
  }
})