Content inspection and archive walks on seeding nodes (relay)
Test and deploy / test (push) Successful in 1m49s
Test and deploy / deploy (push) Successful in 2m15s

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 11:17:02 +02:00
1 parent 51975fe603
commit 2dbca9b152
22 files changed
+1198 -540

No files matched your search

+88 -1
View File
@@ -11,6 +11,9 @@ import { segmentFromHeader } from '#lib/ingest/segment'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
import type { IngestSegment } from '#lib/ingest/segment'
import FileStore from '#services/FileStore'
import RemoteContentService from '#services/RemoteContentService'
import { Job } from 'bullmq'
function header(subject: string, n: number) {
return { 'subject': subject, 'from': 'api seeder <api@example.invalid>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<api-${n}-${subject.length}@test>`, ':bytes': '500000' }
@@ -25,7 +28,10 @@ function batch(next: number): SeedBatch {
test.group('Seed API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
group.teardown(async () => {
await RemoteContentService.close()
await QueueService.closeAll()
})
test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('test-node')
@@ -128,6 +134,87 @@ test.group('Seed API', (group) => {
}
})
test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [file] = await FileStore.insert([
{
filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar',
file_key: 'inspect-node-file'.padEnd(40, '0'),
poster: 'x',
date: now.getTime(),
parts: 2,
groups: '[]',
segments: [
{ number: 1, id: 'one@test', size: 100 },
{ number: 2, id: 'two@test', size: 100 },
],
release_id: release.id,
created_at: now,
updated_at: now,
},
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)
leased.assertStatus(200)
const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id)
assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test'])
// Locked: nobody else gets it meanwhile.
;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204)
// What probeInspection found in the first segment; posted as a JSON string, empty strings and all.
const result = {
size: 15_728_640,
contentType: null,
md5_16k: null,
yencName: '',
archive: {
format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120,
entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }],
},
}
const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) }
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200)
const row = await db.from('releases').where('id', release.id).first()
assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP')
assert.isTrue(row.encrypted)
assert.lengthOf(row.contents, 1)
assert.isOk((await db.from('files').where('id', file.id).first()).archive)
assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
// Too late (or twice): the lock is gone.
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409)
})
test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node-2')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now },
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
const first = await lease()
;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200)
const again = await lease()
assert.equal(again.jobId, first.jobId)
;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: '430 no such article' }).headers(auth)).assertStatus(200)
// Retried later (backoff), like a local failure.
assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState())
})
test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('late-node')
const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id')