import { test } from '@japa/runner' import { gzipSync } from 'node:zlib' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import SeedOrchestrator from '#services/SeedOrchestrator' import QueueService from '#services/QueueService' import indexerConfig from '#config/indexer' import SeedFetchPool from '#services/SeedFetchPool' import { deflateSync } from 'node:zlib' import { segmentFromHeader } from '#lib/ingest/segment' import { FileAggregator } from '#lib/aggregate/index' import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire' import type { IngestSegment } from '#lib/ingest/segment' import FileStore from '#services/FileStore' import RemoteContentService from '#services/RemoteContentService' import { Job } from 'bullmq' function header(subject: string, n: number) { return { 'subject': subject, 'from': 'api seeder ', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': ``, ':bytes': '500000' } } function batch(next: number): SeedBatch { const segments = [1, 2].map((n) => segmentFromHeader(header(`Api.Test.2026 - "Api.Test.2026.mkv" yEnc (${n}/2)`, n), 'alt.binaries.test', n)!) const lone = segmentFromHeader(header('Api.Test.2026 - "Api.Test.2026.nfo" yEnc (1/3)', 9), 'alt.binaries.test', 9)! const aggregator = new FileAggregator() return { next, headers: 3, obfuscated: 0, deferred: 0, files: aggregator.add(segments).map(toSeedFile), fragments: [toFragment(lone)] } } test.group('Seed API', (group) => { group.each.setup(() => testUtils.db().withGlobalTransaction()) group.teardown(async () => { await RemoteContentService.close() await QueueService.closeAll() }) test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('test-node') await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }) const auth = { authorization: `Bearer ${token}` } ;(await client.post('/api/v1/seed/lease')).assertStatus(401) const leased = await client.post('/api/v1/seed/lease').headers(auth) leased.assertStatus(200) const range = leased.body().range assert.include(range, { group: 'alt.binaries.test', start: 1, end: 100 }) // As seed:node sends it: a gzip-compressed JSON body. const posted = await fetch(`http://${process.env.HOST}:${process.env.PORT}/api/v1/seed/ranges/${range.id}/batches`, { method: 'POST', headers: { ...auth, 'content-type': 'application/json', 'content-encoding': 'gzip' }, body: gzipSync(JSON.stringify(batch(51))), }) assert.equal(posted.status, 200) assert.include(((await posted.json()) as any).stored, { files: 1 }) const file = await db.from('files').where('filename', 'Api.Test.2026.mkv').first() assert.equal(file.parts, 2) // The lone .nfo segment sits near the range's edge: kept as a binary (not staged) for the neighbouring range. assert.isOk(await db.from('binaries').where('name', 'Api.Test.2026.nfo').first()) const row = await db.from('seed_ranges').where('id', range.id).first() assert.equal(Number(row.next_id), 51) assert.equal(Number(row.headers), 3) ;(await client.post(`/api/v1/seed/ranges/${range.id}/done`).headers(auth)).assertStatus(200) assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done') ;(await client.post('/api/v1/seed/lease').headers(auth)).assertStatus(204) }) test('a quiet lease passes to another node, and the first one gets 409', async ({ client, assert }) => { const first = await SeedOrchestrator.issueToken('node-a') const second = await SeedOrchestrator.issueToken('node-b') await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }) const range = (await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${first}` })).body().range await db.from('seed_ranges').where('id', range.id).update({ next_id: 40, heartbeat_at: new Date(Date.now() - 60 * 60 * 1000) }) const taken = (await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${second}` })).body().range assert.equal(taken.id, range.id) assert.equal(taken.start, 40) const late = await client.post(`/api/v1/seed/ranges/${range.id}/batches`).headers({ authorization: `Bearer ${first}` }).json(batch(60)) late.assertStatus(409) ;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers({ authorization: `Bearer ${second}` }).json({ error: 'test' })).assertStatus(200) assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'queued') }) test('no leases while the database is over the disk limit', async ({ client }) => { const token = await SeedOrchestrator.issueToken('full-disk') await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }) const limit = indexerConfig.backfill.maxDefaultGb indexerConfig.backfill.maxDefaultGb = 0 try { ;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204) } finally { indexerConfig.backfill.maxDefaultGb = limit } ;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(200) }) test('a range given back before it started does not use up an attempt', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('no-news-server') await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }) const auth = { authorization: `Bearer ${token}` } const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range ;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers(auth).json({ error: 'NNTP login: 502', uncounted: true })).assertStatus(200) const row = await db.from('seed_ranges').where('id', range.id).first() assert.equal(row.status, 'queued') assert.equal(row.attempts, 0) }) test('relay: a leased range gets its overview chunks through the API', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('relay-node') await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }) const auth = { authorization: `Bearer ${token}` } const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range const fetch = SeedFetchPool.fetch const connections = indexerConfig.seed.nntpConnections const asked: unknown[] = [] SeedFetchPool.fetch = async (group, from, to) => { asked.push([group, from, to]) return deflateSync(Buffer.from('5\tsubject\r\n')) } try { indexerConfig.seed.nntpConnections = 0 ;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 1, to: 50 }).headers(auth)).assertStatus(503) indexerConfig.seed.nntpConnections = 1 const chunk = await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 50, to: 500 }).headers(auth) chunk.assertStatus(200) // Clamped to the range. assert.deepEqual(asked, [['alt.binaries.test', 50, 100]]) ;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 200, to: 300 }).headers(auth)).assertStatus(400) } finally { SeedFetchPool.fetch = fetch indexerConfig.seed.nntpConnections = connections } }) test('ranges of a higher priority are leased first', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('priority-node') await db.table('seed_ranges').insert([ { group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }, { group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101, priority: 1 }, ]) const leased = await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` }) assert.include(leased.body().range, { start: 101, end: 200 }) }) test('priorities take turns: half the leases go to the higher priority', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('turns-node') const auth = { authorization: `Bearer ${token}` } await db.table('seed_ranges').insert([ { group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }, { group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101 }, { group_name: 'alt.binaries.test', start_id: 1001, end_id: 1100, next_id: 1001, priority: 1 }, { group_name: 'alt.binaries.test', start_id: 1101, end_id: 1200, next_id: 1101, priority: 1 }, ]) const starts: number[] = [] for (let i = 0; i < 4; i++) starts.push((await client.post('/api/v1/seed/lease').headers(auth)).body().range.start) assert.deepEqual(starts, [1001, 1, 1101, 101]) }) test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => { await QueueService.collectionQueue.drain() await QueueService.archiveWalkQueue.drain() const token = await SeedOrchestrator.issueToken('inspect-node') const auth = { authorization: `Bearer ${token}` } const now = new Date() const [release] = await db .table('releases') .insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }) .returning('id') const [file] = await FileStore.insert([ { filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar', file_key: 'inspect-node-file'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 2, groups: '[]', segments: [ { number: 1, id: 'one@test', size: 100 }, { number: 2, id: 'two@test', size: 100 }, ], release_id: release.id, created_at: now, updated_at: now, }, ]) await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` }) const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth) leased.assertStatus(200) const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id) assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test']) // Locked: nobody else gets it meanwhile. ;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204) // What probeInspection found in the first segment; posted as a JSON string, empty strings and all. const result = { size: 15_728_640, contentType: null, md5_16k: null, yencName: '', archive: { format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120, entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }], }, } const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) } ;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200) const row = await db.from('releases').where('id', release.id).first() assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP') assert.isTrue(row.encrypted) assert.lengthOf(row.contents, 1) assert.isOk((await db.from('file_details').where('file_id', file.id).first())?.archive) assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed') // Too late (or twice): the lock is gone. ;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409) }) test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => { await QueueService.collectionQueue.drain() await QueueService.archiveWalkQueue.drain() const token = await SeedOrchestrator.issueToken('inspect-node-2') const auth = { authorization: `Bearer ${token}` } const now = new Date() const [file] = await FileStore.insert([ { filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now }, ]) await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` }) const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0] const first = await lease() ;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200) const again = await lease() assert.equal(again.jobId, first.jobId) ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: 'NNTP BODY: no answer within 120 s' }).headers(auth)).assertStatus(200) // Retried later (backoff), like a local failure. assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState()) }) test('a missing first article reported by a node is recorded, not retried', async ({ client, assert }) => { await QueueService.collectionQueue.drain() await QueueService.archiveWalkQueue.drain() const token = await SeedOrchestrator.issueToken('inspect-node-3') const auth = { authorization: `Bearer ${token}` } const now = new Date() const [file] = await FileStore.insert([ { filename: 'Wq7n3LpX9vT2rK8mZ5cYbA.bin', file_key: 'inspect-node-file-3'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'gone@test', size: 1 }], created_at: now, updated_at: now }, ]) await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` }) const job = (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0] const error = 'Seed API /api/v1/seed/articles?id=gone%40test: 410 {"error":"No such article"}' ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: job.queue, jobId: job.jobId, token: job.token, error }).headers(auth)).assertStatus(200) assert.equal((await db.from('file_details').where('file_id', file.id).first())?.content_type, 'missing') assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed') }) test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('late-node') const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id') ;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204) assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done') }) })