Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/seed_api.spec.ts

@@ -11,6 +11,9 @@ import { segmentFromHeader } from '#lib/ingest/segment'
 import { FileAggregator } from '#lib/aggregate/index'
 import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
 import type { IngestSegment } from '#lib/ingest/segment'
+import FileStore from '#services/FileStore'
+import RemoteContentService from '#services/RemoteContentService'
+import { Job } from 'bullmq'
 
 function header(subject: string, n: number) {
   return { 'subject': subject, 'from': 'api seeder <api@example.invalid>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<api-${n}-${subject.length}@test>`, ':bytes': '500000' }
@@ -25,7 +28,10 @@ function batch(next: number): SeedBatch {
 
 test.group('Seed API', (group) => {
   group.each.setup(() => testUtils.db().withGlobalTransaction())
-  group.teardown(() => QueueService.closeAll())
+  group.teardown(async () => {
+    await RemoteContentService.close()
+    await QueueService.closeAll()
+  })
 
   test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => {
     const token = await SeedOrchestrator.issueToken('test-node')
@@ -128,6 +134,87 @@ test.group('Seed API', (group) => {
     }
   })
 
+  test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
+    await QueueService.collectionQueue.drain()
+    await QueueService.archiveWalkQueue.drain()
+    const token = await SeedOrchestrator.issueToken('inspect-node')
+    const auth = { authorization: `Bearer ${token}` }
+    const now = new Date()
+    const [release] = await db
+      .table('releases')
+      .insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+      .returning('id')
+    const [file] = await FileStore.insert([
+      {
+        filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar',
+        file_key: 'inspect-node-file'.padEnd(40, '0'),
+        poster: 'x',
+        date: now.getTime(),
+        parts: 2,
+        groups: '[]',
+        segments: [
+          { number: 1, id: 'one@test', size: 100 },
+          { number: 2, id: 'two@test', size: 100 },
+        ],
+        release_id: release.id,
+        created_at: now,
+        updated_at: now,
+      },
+    ])
+    await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
+
+    const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)
+    leased.assertStatus(200)
+    const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id)
+    assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test'])
+    // Locked: nobody else gets it meanwhile.
+    ;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204)
+
+    // What probeInspection found in the first segment; posted as a JSON string, empty strings and all.
+    const result = {
+      size: 15_728_640,
+      contentType: null,
+      md5_16k: null,
+      yencName: '',
+      archive: {
+        format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120,
+        entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }],
+      },
+    }
+    const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) }
+    ;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200)
+
+    const row = await db.from('releases').where('id', release.id).first()
+    assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP')
+    assert.isTrue(row.encrypted)
+    assert.lengthOf(row.contents, 1)
+    assert.isOk((await db.from('files').where('id', file.id).first()).archive)
+    assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
+    // Too late (or twice): the lock is gone.
+    ;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409)
+  })
+
+  test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => {
+    await QueueService.collectionQueue.drain()
+    await QueueService.archiveWalkQueue.drain()
+    const token = await SeedOrchestrator.issueToken('inspect-node-2')
+    const auth = { authorization: `Bearer ${token}` }
+    const now = new Date()
+    const [file] = await FileStore.insert([
+      { filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now },
+    ])
+    await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
+    const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
+
+    const first = await lease()
+    ;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200)
+    const again = await lease()
+    assert.equal(again.jobId, first.jobId)
+    ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: '430 no such article' }).headers(auth)).assertStatus(200)
+    // Retried later (backoff), like a local failure.
+    assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState())
+  })
+
   test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => {
     const token = await SeedOrchestrator.issueToken('late-node')
     const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id')

tests/unit/probe.spec.ts

@@ -0,0 +1,78 @@
+import { test } from '@japa/runner'
+import { crc32Hex } from '#lib/yenc/index'
+import { probeInspection, type InspectTask } from '#lib/content/probe'
+
+/** One yEnc article of `file`: part `index` of `size` bytes (escaping '.' and whitespace too, so no dot-stuffing). */
+function yencArticle(name: string, file: Buffer, index: number, size: number) {
+  const data = file.subarray(index * size, (index + 1) * size)
+  const total = Math.ceil(file.length / size)
+  const out: number[] = []
+  for (const byte of data) {
+    const c = (byte + 42) & 0xff
+    if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
+    else out.push(c)
+  }
+  const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
+  const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
+  return Buffer.from(head + Buffer.from(out).toString('latin1') + tail, 'latin1')
+}
+
+/** A posted file as articles, and its inspection task. */
+function posted(name: string, file: Buffer, size = 400) {
+  const articles = new Map<string, Buffer>()
+  const segments = []
+  for (let i = 0; i * size < file.length; i++) {
+    const id = `${name}-${i}@test`
+    articles.set(id, yencArticle(name, file, i, size))
+    segments.push({ number: i + 1, id, size: size + 100 })
+  }
+  const fetched: string[] = []
+  const fetch = async (id: string) => {
+    fetched.push(id)
+    const body = articles.get(id)
+    if (!body) throw new Error(`430 no such article ${id}`)
+    return body
+  }
+  const task: InspectTask = { kind: 'inspect', fileId: 1, name, segments, sevenZip: { volumes: 1, last: segments } }
+  return { task, fetch, fetched }
+}
+
+// `7z a -pSecret -mhe=on`: an encrypted header (tests/unit/sevenzip.spec.ts).
+const HEADER_ENCRYPTED = Buffer.from(
+  'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
+  'base64'
+)
+
+test.group('content probe', () => {
+  test('a first segment gives size, fingerprint and the yEnc name', async ({ assert }) => {
+    const file = Buffer.alloc(20_000, 7)
+    const { task, fetch, fetched } = posted('Some.Name.bin', file, 17_000)
+    const probe = await probeInspection(task, fetch)
+    assert.include(probe, { size: 20_000, contentType: null, yencName: 'Some.Name.bin' })
+    assert.match(probe.md5_16k!, /^[0-9a-f]{32}$/)
+    assert.deepEqual(fetched, ['Some.Name.bin-0@test'])
+  })
+
+  test('an NZB is read whole', async ({ assert }) => {
+    const xml = `<?xml version="1.0"?>\n<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${'<file subject="x"/>'.repeat(60)}</nzb>`
+    const { task, fetch, fetched } = posted('x.nzb', Buffer.from(xml))
+    const probe = await probeInspection(task, fetch)
+    assert.equal(probe.contentType, 'nzb')
+    assert.equal(probe.text, xml)
+    assert.lengthOf(fetched, task.segments.length + 1) // the first segment twice: once to sniff, once in the whole
+  })
+
+  test('a 7z archive with encrypted headers is known from its tail, without bsdtar', async ({ assert }) => {
+    const { task, fetch } = posted('a.7z', HEADER_ENCRYPTED, 100)
+    const probe = await probeInspection(task, fetch)
+    assert.equal(probe.contentType, '7z')
+    assert.include(probe.archive!, { format: '7z', headersEncrypted: true })
+    assert.lengthOf(probe.archive!.entries, 0)
+  })
+
+  test('a missing article fails the probe (the job is retried)', async ({ assert }) => {
+    const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1))
+    task.segments[0].id = 'missing@test'
+    await assert.rejects(() => probeInspection(task, fetch), /430/)
+  })
+})

tests/unit/rar.spec.ts

@@ -325,7 +325,7 @@ test.group('rar / archive walks', () => {
   })
 
   test('finds segments from their yEnc offsets', async ({ assert }) => {
-    const { SegmentReader } = await import('#services/ArchiveWalkService')
+    const { SegmentReader } = await import('#lib/rar/index')
     const set = rar5Set()
     const size = 700
     const articles = new Map<string, Buffer>()
@@ -336,7 +336,7 @@ test.group('rar / archive walks', () => {
         articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
         segments.push({ number: i + 1, id, size: 0 })
       }
-      return { id: v + 1, segments }
+      return segments
     })
     const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
     const walk = await walkRar(reader)