Content inspection and archive walks on seeding nodes (relay)

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/probe.spec.ts

@@ -0,0 +1,78 @@
+import { test } from '@japa/runner'
+import { crc32Hex } from '#lib/yenc/index'
+import { probeInspection, type InspectTask } from '#lib/content/probe'
+
+/** One yEnc article of `file`: part `index` of `size` bytes (escaping '.' and whitespace too, so no dot-stuffing). */
+function yencArticle(name: string, file: Buffer, index: number, size: number) {
+  const data = file.subarray(index * size, (index + 1) * size)
+  const total = Math.ceil(file.length / size)
+  const out: number[] = []
+  for (const byte of data) {
+    const c = (byte + 42) & 0xff
+    if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
+    else out.push(c)
+  }
+  const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
+  const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
+  return Buffer.from(head + Buffer.from(out).toString('latin1') + tail, 'latin1')
+}
+
+/** A posted file as articles, and its inspection task. */
+function posted(name: string, file: Buffer, size = 400) {
+  const articles = new Map<string, Buffer>()
+  const segments = []
+  for (let i = 0; i * size < file.length; i++) {
+    const id = `${name}-${i}@test`
+    articles.set(id, yencArticle(name, file, i, size))
+    segments.push({ number: i + 1, id, size: size + 100 })
+  }
+  const fetched: string[] = []
+  const fetch = async (id: string) => {
+    fetched.push(id)
+    const body = articles.get(id)
+    if (!body) throw new Error(`430 no such article ${id}`)
+    return body
+  }
+  const task: InspectTask = { kind: 'inspect', fileId: 1, name, segments, sevenZip: { volumes: 1, last: segments } }
+  return { task, fetch, fetched }
+}
+
+// `7z a -pSecret -mhe=on`: an encrypted header (tests/unit/sevenzip.spec.ts).
+const HEADER_ENCRYPTED = Buffer.from(
+  'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
+  'base64'
+)
+
+test.group('content probe', () => {
+  test('a first segment gives size, fingerprint and the yEnc name', async ({ assert }) => {
+    const file = Buffer.alloc(20_000, 7)
+    const { task, fetch, fetched } = posted('Some.Name.bin', file, 17_000)
+    const probe = await probeInspection(task, fetch)
+    assert.include(probe, { size: 20_000, contentType: null, yencName: 'Some.Name.bin' })
+    assert.match(probe.md5_16k!, /^[0-9a-f]{32}$/)
+    assert.deepEqual(fetched, ['Some.Name.bin-0@test'])
+  })
+
+  test('an NZB is read whole', async ({ assert }) => {
+    const xml = `<?xml version="1.0"?>\n<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${'<file subject="x"/>'.repeat(60)}</nzb>`
+    const { task, fetch, fetched } = posted('x.nzb', Buffer.from(xml))
+    const probe = await probeInspection(task, fetch)
+    assert.equal(probe.contentType, 'nzb')
+    assert.equal(probe.text, xml)
+    assert.lengthOf(fetched, task.segments.length + 1) // the first segment twice: once to sniff, once in the whole
+  })
+
+  test('a 7z archive with encrypted headers is known from its tail, without bsdtar', async ({ assert }) => {
+    const { task, fetch } = posted('a.7z', HEADER_ENCRYPTED, 100)
+    const probe = await probeInspection(task, fetch)
+    assert.equal(probe.contentType, '7z')
+    assert.include(probe.archive!, { format: '7z', headersEncrypted: true })
+    assert.lengthOf(probe.archive!.entries, 0)
+  })
+
+  test('a missing article fails the probe (the job is retried)', async ({ assert }) => {
+    const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1))
+    task.segments[0].id = 'missing@test'
+    await assert.rejects(() => probeInspection(task, fetch), /430/)
+  })
+})

tests/unit/rar.spec.ts

@@ -325,7 +325,7 @@ test.group('rar / archive walks', () => {
   })
 
   test('finds segments from their yEnc offsets', async ({ assert }) => {
-    const { SegmentReader } = await import('#services/ArchiveWalkService')
+    const { SegmentReader } = await import('#lib/rar/index')
     const set = rar5Set()
     const size = 700
     const articles = new Map<string, Buffer>()
@@ -336,7 +336,7 @@ test.group('rar / archive walks', () => {
         articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
         segments.push({ number: i + 1, id, size: 0 })
       }
-      return { id: v + 1, segments }
+      return segments
     })
     const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
     const walk = await walkRar(reader)