Content inspection and archive walks on seeding nodes (relay)
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51975fe603
commit
2dbca9b152
22 files changed
+1198
-540
No files matched your search
@@ -11,6 +11,9 @@ import { segmentFromHeader } from '#lib/ingest/segment'
|
||||
import { FileAggregator } from '#lib/aggregate/index'
|
||||
import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
|
||||
import type { IngestSegment } from '#lib/ingest/segment'
|
||||
import FileStore from '#services/FileStore'
|
||||
import RemoteContentService from '#services/RemoteContentService'
|
||||
import { Job } from 'bullmq'
|
||||
|
||||
function header(subject: string, n: number) {
|
||||
return { 'subject': subject, 'from': 'api seeder <api@example.invalid>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<api-${n}-${subject.length}@test>`, ':bytes': '500000' }
|
||||
@@ -25,7 +28,10 @@ function batch(next: number): SeedBatch {
|
||||
|
||||
test.group('Seed API', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
group.teardown(() => QueueService.closeAll())
|
||||
group.teardown(async () => {
|
||||
await RemoteContentService.close()
|
||||
await QueueService.closeAll()
|
||||
})
|
||||
|
||||
test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => {
|
||||
const token = await SeedOrchestrator.issueToken('test-node')
|
||||
@@ -128,6 +134,87 @@ test.group('Seed API', (group) => {
|
||||
}
|
||||
})
|
||||
|
||||
test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
|
||||
await QueueService.collectionQueue.drain()
|
||||
await QueueService.archiveWalkQueue.drain()
|
||||
const token = await SeedOrchestrator.issueToken('inspect-node')
|
||||
const auth = { authorization: `Bearer ${token}` }
|
||||
const now = new Date()
|
||||
const [release] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
const [file] = await FileStore.insert([
|
||||
{
|
||||
filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar',
|
||||
file_key: 'inspect-node-file'.padEnd(40, '0'),
|
||||
poster: 'x',
|
||||
date: now.getTime(),
|
||||
parts: 2,
|
||||
groups: '[]',
|
||||
segments: [
|
||||
{ number: 1, id: 'one@test', size: 100 },
|
||||
{ number: 2, id: 'two@test', size: 100 },
|
||||
],
|
||||
release_id: release.id,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
])
|
||||
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
|
||||
|
||||
const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)
|
||||
leased.assertStatus(200)
|
||||
const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id)
|
||||
assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test'])
|
||||
// Locked: nobody else gets it meanwhile.
|
||||
;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204)
|
||||
|
||||
// What probeInspection found in the first segment; posted as a JSON string, empty strings and all.
|
||||
const result = {
|
||||
size: 15_728_640,
|
||||
contentType: null,
|
||||
md5_16k: null,
|
||||
yencName: '',
|
||||
archive: {
|
||||
format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120,
|
||||
entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }],
|
||||
},
|
||||
}
|
||||
const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) }
|
||||
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200)
|
||||
|
||||
const row = await db.from('releases').where('id', release.id).first()
|
||||
assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP')
|
||||
assert.isTrue(row.encrypted)
|
||||
assert.lengthOf(row.contents, 1)
|
||||
assert.isOk((await db.from('files').where('id', file.id).first()).archive)
|
||||
assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
|
||||
// Too late (or twice): the lock is gone.
|
||||
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409)
|
||||
})
|
||||
|
||||
test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => {
|
||||
await QueueService.collectionQueue.drain()
|
||||
await QueueService.archiveWalkQueue.drain()
|
||||
const token = await SeedOrchestrator.issueToken('inspect-node-2')
|
||||
const auth = { authorization: `Bearer ${token}` }
|
||||
const now = new Date()
|
||||
const [file] = await FileStore.insert([
|
||||
{ filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now },
|
||||
])
|
||||
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
|
||||
const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
|
||||
|
||||
const first = await lease()
|
||||
;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200)
|
||||
const again = await lease()
|
||||
assert.equal(again.jobId, first.jobId)
|
||||
;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: '430 no such article' }).headers(auth)).assertStatus(200)
|
||||
// Retried later (backoff), like a local failure.
|
||||
assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState())
|
||||
})
|
||||
|
||||
test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => {
|
||||
const token = await SeedOrchestrator.issueToken('late-node')
|
||||
const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id')
|
||||
|
||||
Reference in new issue
Block a user