Seed relay: nodes get their headers through the seed API
Test and deploy / test (push) Successful in 1m28s
Test and deploy / deploy (push) Successful in 2m5s

The seed API keeps a pool of NNTP connections (SEED_NNTP_CONNECTIONS) and
serves GET /api/v1/seed/ranges/:id/chunk?from=&to= for leased ranges:
the gzip stream the news server sent, passed on unopened. seed:node fetches
that way by default (SEED_FETCH=relay), so a node needs only a token and
SEED_CONNECTIONS; SEED_FETCH=direct keeps its own NNTP connections. All
fetching then comes from the indexer's IP (the account allows two).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-05 15:24:39 +02:00
1 parent 3d5cfb2527
commit 41859636d3
13 files changed
+289 -33

No files matched your search

+29
View File
@@ -5,6 +5,8 @@ import testUtils from '@adonisjs/core/services/test_utils'
import SeedOrchestrator from '#services/SeedOrchestrator'
import QueueService from '#services/QueueService'
import indexerConfig from '#config/indexer'
import SeedFetchPool from '#services/SeedFetchPool'
import { deflateSync } from 'node:zlib'
import { segmentFromHeader } from '#lib/ingest/segment'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
@@ -99,4 +101,31 @@ test.group('Seed API', (group) => {
assert.equal(row.status, 'queued')
assert.equal(row.attempts, 0)
})
test('relay: a leased range gets its overview chunks through the API', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('relay-node')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range
const fetch = SeedFetchPool.fetch
const connections = indexerConfig.seed.nntpConnections
const asked: unknown[] = []
SeedFetchPool.fetch = async (group, from, to) => {
asked.push([group, from, to])
return deflateSync(Buffer.from('5\tsubject\r\n'))
}
try {
indexerConfig.seed.nntpConnections = 0
;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 1, to: 50 }).headers(auth)).assertStatus(503)
indexerConfig.seed.nntpConnections = 1
const chunk = await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 50, to: 500 }).headers(auth)
chunk.assertStatus(200)
// Clamped to the range.
assert.deepEqual(asked, [['alt.binaries.test', 50, 100]])
;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 200, to: 300 }).headers(auth)).assertStatus(400)
} finally {
SeedFetchPool.fetch = fetch
indexerConfig.seed.nntpConnections = connections
}
})
})
+17
View File
@@ -2,6 +2,7 @@ import { test } from '@japa/runner'
import net from 'node:net'
import { deflateSync } from 'node:zlib'
import { XoverClient, parseOverviews } from '#lib/nntp/xover_client'
import { unzipSync } from 'node:zlib'
const LINES = [
'101\tShow.S01E01 - "a.mkv" yEnc (1/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<one@test>\t\t700000\t5000\tXref: news.example a.b.test:101',
@@ -86,4 +87,20 @@ test.group('nntp / XoverClient', () => {
close()
}
})
for (const compress of [false, true]) {
test(`hands out overviews as one zlib stream (${compress ? 'as the server sent it' : 'deflated here'})`, async ({ assert }) => {
const { port, close } = await server()
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
try {
await client.group('a.b.test')
const raw = await client.xoverRaw(101, 102)
assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [101, 102])
assert.lengthOf(parseOverviews(unzipSync(await client.xoverRaw(1, 1)).toString('utf8')), 0)
} finally {
client.close()
close()
}
})
}
})