Complete archive listings by walking RAR headers, at low priority

Inspection lists only the entries whose headers are in the first segment of a first volume. A walk hops from header to header, fetching one segment per entry and volume, in its own queue (one job per collection worker) on NNTP connections inspection isn't waiting for. content:walk-archives queues releases inspected earlier (CONCEPT D40). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 23:32:52 +02:00
1 parent b7ae8267c6
commit f4ded062df
@@ -1,6 +1,7 @@
 import { test } from '@japa/runner'
 import { crc32 } from 'node:zlib'
-import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, type RarEntry } from '#lib/rar/index'
+import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, walkRar, type RarEntry, type VolumeReader } from '#lib/rar/index'
+import { crc32Hex } from '#lib/yenc/index'

 /** Entries without the extraction fields, for comparing listings. */
 const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
@@ -228,3 +229,135 @@ test.group('rar / looksRandom capitals', () => {
    assert.isFalse(looksRandom('TheQuickBrownFoxJumps'))
  })
 })
+
+// --- Archive walks (D40) --------------------------------------------------------
+
+/** Volumes split into segments of `segmentSize` bytes; counts the segments read. */
+function memoryReader(volumes: Buffer[], segmentSize: number) {
+  const read = new Set<string>()
+  const reader: VolumeReader & { segments: Set<string> } = {
+    segments: read,
+    volumes: volumes.length,
+    size: (v) => volumes[v].length,
+    async read(v, offset, more) {
+      const index = Math.floor(offset / segmentSize)
+      if (offset >= volumes[v].length) return null
+      read.add(`${v}:${index}`)
+      if (more) read.add(`${v}:${index + 1}`)
+      return volumes[v].subarray(offset, (index + (more ? 2 : 1)) * segmentSize)
+    },
+  }
+  return reader
+}
+
+function rar5Set() {
+  const main = (n: number) => rar5Block(1, 0, n === 0 ? vint(0x0001) : Buffer.concat([vint(0x0003), vint(n)]))
+  const end = rar5Block(5, 0, vint(0x0001))
+  const SPLIT_BEFORE = 0x0008
+  const SPLIT_AFTER = 0x0010
+  return [
+    Buffer.concat([RAR5, main(0), rar5File('Book/01.mp3', 3000, { packed: 3000 }), rar5File('Book/02.mp3', 7000, { packed: 5000, headerFlags: SPLIT_AFTER }), end]),
+    Buffer.concat([
+      RAR5,
+      main(1),
+      rar5File('Book/02.mp3', 7000, { packed: 2000, headerFlags: SPLIT_BEFORE }),
+      rar5File('Book/03.mp3', 1000, { packed: 1000 }),
+      rar5File('Book/04.mp3', 4500, { packed: 4000, headerFlags: SPLIT_AFTER }),
+      end,
+    ]),
+    Buffer.concat([RAR5, main(2), rar5File('Book/04.mp3', 4500, { packed: 500, headerFlags: SPLIT_BEFORE }), rar5File('Book/05.mp3', 100, { packed: 100 }), rar5Block(5, 0, vint(0))]),
+  ]
+}
+
+test.group('rar / archive walks', () => {
+  test('lists every entry of a RAR5 set, reading a segment per header', async ({ assert }) => {
+    // 700-byte segments: headers get cut off by segment ends.
+    const reader = memoryReader(rar5Set(), 700)
+    const walk = await walkRar(reader)
+    assert.isTrue(walk.complete)
+    assert.equal(walk.format, 'rar5')
+    assert.deepEqual(walk.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3', 'Book/03.mp3', 'Book/04.mp3', 'Book/05.mp3'])
+    assert.equal(walk.entries[1].size, 7000)
+    const total = rar5Set().reduce((sum, v) => sum + Math.ceil(v.length / 700), 0)
+    assert.isBelow(reader.segments.size, total / 2)
+  })
+
+  test('lists a RAR4 set across volumes', async ({ assert }) => {
+    const end = rar4Block(0x7b, 0, Buffer.alloc(0))
+    const volumes = [
+      Buffer.concat([RAR4, rar4Main(0x0001 | 0x0100), rar4File(Buffer.from('a.flac'), { packed: 900, unpacked: 900 }), rar4File(Buffer.from('b.flac'), { packed: 2000, unpacked: 2500, flags: 0x0002 }), end]),
+      Buffer.concat([RAR4, rar4Main(0x0001), rar4File(Buffer.from('b.flac'), { packed: 500, unpacked: 2500, flags: 0x0001 }), rar4File(Buffer.from('c.flac'), { packed: 10, unpacked: 10 }), end]),
+    ]
+    const walk = await walkRar(memoryReader(volumes, 512))
+    assert.isTrue(walk.complete)
+    assert.deepEqual(walk.entries.map((e) => e.name), ['a.flac', 'b.flac', 'c.flac'])
+  })
+
+  test('stops at a missing segment or a broken header, keeping what it found', async ({ assert }) => {
+    const set = rar5Set()
+    const missing = memoryReader(set, 700)
+    const read = missing.read.bind(missing)
+    missing.read = async (v, offset, more) => (v === 1 ? null : read(v, offset, more))
+    const partial = await walkRar(missing)
+    assert.equal(partial.stopped, 'unreadable')
+    assert.deepEqual(partial.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3'])
+
+    const broken = rar5Set()
+    // The header of 03.mp3 in volume 2, after 02.mp3's continuation and its data.
+    const at = broken[1].indexOf(Buffer.from('Book/03.mp3')) - 4
+    broken[1][at] ^= 0xff
+    const invalid = await walkRar(memoryReader(broken, 700))
+    assert.equal(invalid.stopped, 'invalid')
+    assert.lengthOf(invalid.entries, 2)
+  })
+
+  test('finds segments from their yEnc offsets', async ({ assert }) => {
+    const { SegmentReader } = await import('#services/ArchiveWalkService')
+    const set = rar5Set()
+    const size = 700
+    const articles = new Map<string, Buffer>()
+    const files = set.map((volume, v) => {
+      const segments = []
+      for (let i = 0; i * size < volume.length; i++) {
+        const id = `v${v}s${i}@test`
+        articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
+        segments.push({ number: i + 1, id, size: 0 })
+      }
+      return { id: v + 1, segments }
+    })
+    const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
+    const walk = await walkRar(reader)
+    assert.isTrue(walk.complete)
+    assert.lengthOf(walk.entries, 5)
+    assert.isBelow(reader.fetched, articles.size / 2)
+
+    const capped = new SegmentReader(files, 3, async (id) => articles.get(id)!)
+    const stopped = await walkRar(capped)
+    assert.equal(stopped.stopped, 'unreadable')
+    assert.equal(capped.fetched, 3)
+  })
+})
+
+/** One yEnc article of `file`: part `index` of `size` bytes. */
+function yencArticle(name: string, file: Buffer, index: number, size: number) {
+  const data = file.subarray(index * size, (index + 1) * size)
+  const total = Math.ceil(file.length / size)
+  const out: number[] = []
+  for (const byte of data) {
+    const c = (byte + 42) & 0xff
+    // Escape the critical characters, and '.' and whitespace so no line needs dot-stuffing.
+    if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
+    else out.push(c)
+  }
+  const lines: string[] = []
+  const encoded = Buffer.from(out)
+  for (let i = 0; i < encoded.length; i += 128) {
+    let end = Math.min(encoded.length, i + 128)
+    if (encoded[end - 1] === 0x3d && end < encoded.length) end++
+    lines.push(encoded.subarray(i, end).toString('latin1'))
+    i = end - 128
+  }
+  const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
+  const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
+  return Buffer.from(head + lines.join('\r\n') + tail, 'latin1')
+}