Probe 7z listings and ZIP starts

7z keeps its file list at the end of the archive, so the 7z-tail
candidates (the last segment of the highest volume) were queued but
never probed, and the .7z.001 start only gave the type. The prober now
claims 7z-tail candidates after the archives; lib/sevenzip finds the end
header in the article, decodes the LZMA-packed header (a TypeScript port
of the LZMA SDK's reference decoder) and lists the files with their
sizes. An AES-encrypted header means passworded without names. Names
feed the inner name and the judge's fake rule.

An unsplit .7z of several segments now also gets a tail candidate, and
.zip files an archive candidate: their first segment's local headers
give the first entries and whether they are encrypted. Both in the Rust
edge and the TypeScript indexer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/edge/range_indexer.spec.ts

@@ -55,6 +55,20 @@ test.group('edge / RangeIndexer', () => {
     assert.equal(batch.next, 1_000_001)
   })
 
+  test("archive candidates: a split 7z's start and its highest volume's end, an unsplit 7z's start and end, a ZIP's start", ({ assert }) => {
+    const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 10 })
+    indexer.add([
+      ...post(100, 'Split.Seven.7z.001', 2, { counter: 'Split.Seven [1/3]' }),
+      ...post(110, 'Split.Seven.7z.003', 2, { counter: 'Split.Seven [3/3]' }),
+      ...post(120, 'Split.Seven.7z.002', 2, { counter: 'Split.Seven [2/3]' }),
+      ...post(200, 'Whole.Seven.7z', 3, { counter: 'Whole.Seven [1/1]' }),
+      ...post(300, 'Some.Zip.zip', 2, { counter: 'Some.Zip [1/1]' }),
+    ], 200_000)
+    indexer.finish()
+    const got = indexer.take().candidates.map((c) => `${c.kind} ${c.file}`).sort()
+    assert.deepEqual(got, ['7z-tail Split.Seven.7z.003', '7z-tail Whole.Seven.7z', 'archive Some.Zip.zip', 'archive Split.Seven.7z.001', 'archive Whole.Seven.7z'])
+  })
+
   test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => {
     const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
     indexer.add([

tests/unit/probe/inspect.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
 import { crc32 } from 'node:zlib'
 import yencode from 'yencode'
 import { crc32Hex } from '#lib/yenc/index'
@@ -148,4 +149,45 @@ test.group('probe / innerName', () => {
     // Nor does packaging: a lone par2 volume.
     assert.isNull(innerName([{ kind: 'lone', result: { yencName: 'a1b2.vol007+08.par2', size: 1, type: 'par2' } }], 'x9y8'))
   })
+
+  test("a 7z-tail: the listing from the last segment of the last volume; one that isn't the last segment: none", ({ assert }) => {
+    const tail = readFileSync(new URL('../../fixtures/sevenzip/split-last-volume.tail', import.meta.url))
+    // The volume's last 8,000 bytes as its third part, of a volume 1,000,000 bytes long.
+    const last = tail.subarray(tail.length - 8000)
+    const part = (data: Buffer, begin: number) =>
+      Buffer.from(
+        [`=ybegin part=3 total=3 line=128 size=1000000 name=x.7z.004`, `=ypart begin=${begin} end=${begin + data.length - 1}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} part=3 pcrc32=${crc32Hex(data)}`].join('\r\n'),
+        'latin1'
+      )
+    const result = inspect(part(last, 1_000_000 - 8000 + 1), '7z-tail')
+    assert.equal(result.entries, 4)
+    assert.deepEqual(result.names?.[0], 'Some.Movie.2020.1080p.BluRay.x264-GRP/some.movie.2020.1080p.bluray.x264-grp.mkv')
+    assert.isFalse(result.passworded)
+    assert.isUndefined(inspect(part(last, 1), '7z-tail').names)
+    const encrypted = readFileSync(new URL('../../fixtures/sevenzip/header-encrypted.tail', import.meta.url))
+    const hidden = inspect(part(encrypted.subarray(encrypted.length - 8000), 1_000_000 - 8000 + 1), '7z-tail')
+    assert.isTrue(hidden.passworded)
+    assert.equal(hidden.entries, 0)
+    assert.isUndefined(hidden.names)
+    // A small 7z whole in one article, probed as an archive start.
+    assert.deepEqual(inspect(article(readFileSync(new URL('../../fixtures/sevenzip/tiny.7z', import.meta.url)), 'x.7z'), 'archive').names, ['b.nfo', 'a.txt'])
+  })
+
+  test('a ZIP start: the names of its first entries, and whether they are encrypted', ({ assert }) => {
+    const local = (name: string, data: Buffer, flags = 0) => {
+      const header = Buffer.alloc(30)
+      header.writeUInt32LE(0x04034b50, 0)
+      header.writeUInt16LE(flags, 6)
+      header.writeUInt32LE(data.length, 18)
+      header.writeUInt32LE(data.length, 22)
+      header.writeUInt16LE(Buffer.byteLength(name), 26)
+      return Buffer.concat([header, Buffer.from(name), data])
+    }
+    const zip = Buffer.concat([local('Album/', Buffer.alloc(0)), local('Album/01 - Track.flac', Buffer.alloc(3000, 1)), local('Album/cover.jpg', Buffer.alloc(500, 2))])
+    const plain = inspect(article(zip, 'album.zip'))
+    assert.equal(plain.type, 'zip')
+    assert.deepEqual(plain.names, ['Album/01 - Track.flac', 'Album/cover.jpg'])
+    assert.isFalse(plain.passworded)
+    assert.isTrue(inspect(article(local('secret.mkv', Buffer.alloc(100), 0x1), 'x.zip')).passworded)
+  })
 })

tests/unit/probe/lzma.spec.ts

@@ -0,0 +1,20 @@
+import { test } from '@japa/runner'
+import { readFileSync } from 'node:fs'
+import { lzmaDecode, LzmaError } from '#lib/sevenzip/lzma'
+
+const fixture = (name: string) => readFileSync(new URL(`../../fixtures/sevenzip/${name}`, import.meta.url))
+
+test.group('probe / LZMA', () => {
+  test('decodes a stream Python made (.lzma: 5 property bytes, the size, the raw stream)', ({ assert }) => {
+    const alone = fixture('sample.lzma')
+    const expected = fixture('sample.bin')
+    const decoded = lzmaDecode(alone.subarray(0, 5), alone.subarray(13), expected.length)
+    assert.isTrue(decoded.equals(expected))
+  })
+
+  test('a cut-off or damaged stream throws LzmaError', ({ assert }) => {
+    const alone = fixture('sample.lzma')
+    const size = fixture('sample.bin').length
+    assert.throws(() => lzmaDecode(alone.subarray(0, 5), alone.subarray(13, 200), size), LzmaError)
+  })
+})

tests/unit/probe/sevenzip.spec.ts

@@ -0,0 +1,45 @@
+import { test } from '@japa/runner'
+import { readFileSync } from 'node:fs'
+import { parseSevenZipTail } from '#lib/sevenzip/index'
+
+const fixture = (name: string) => readFileSync(new URL(`../../fixtures/sevenzip/${name}`, import.meta.url))
+const DIR = 'Some.Movie.2020.1080p.BluRay.x264-GRP'
+
+test.group('probe / 7z listing from the archive end', () => {
+  test('an LZMA-packed header (7z default): files with sizes, directories left out', ({ assert }) => {
+    const listing = parseSevenZipTail(fixture('plain.tail'))
+    assert.deepEqual(listing, {
+      files: [
+        { name: `${DIR}/empty.txt`, size: 0 },
+        { name: `${DIR}/Sample/sample.mkv`, size: 20000 },
+        { name: `${DIR}/grp.nfo`, size: 4 },
+        { name: `${DIR}/some.movie.2020.1080p.bluray.x264-grp.mkv`, size: 300000 },
+      ],
+      encrypted: false,
+      headerEncrypted: false,
+    })
+  })
+
+  test('an uncompressed header (7z -mhc=off) and the last volume of a split archive', ({ assert }) => {
+    for (const name of ['uncompressed-header.tail', 'split-last-volume.tail']) {
+      const listing = parseSevenZipTail(fixture(name))!
+      assert.lengthOf(listing.files, 4, name)
+      assert.isFalse(listing.encrypted, name)
+    }
+  })
+
+  test('encrypted entries: names listed, encrypted; an encrypted header (7z -mhe): no names, encrypted', ({ assert }) => {
+    const files = parseSevenZipTail(fixture('files-encrypted.tail'))!
+    assert.lengthOf(files.files, 4)
+    assert.isTrue(files.encrypted)
+    assert.isFalse(files.headerEncrypted)
+    assert.deepEqual(parseSevenZipTail(fixture('header-encrypted.tail')), { files: [], encrypted: true, headerEncrypted: true })
+  })
+
+  test('a whole small archive; not the end of an archive: null', ({ assert }) => {
+    assert.deepEqual(parseSevenZipTail(fixture('tiny.7z'))?.files.map((f) => f.name), ['a.txt', 'b.nfo'])
+    // The end cut off, or data that isn't 7z at all.
+    assert.isNull(parseSevenZipTail(fixture('plain.tail').subarray(0, 30000)))
+    assert.isNull(parseSevenZipTail(fixture('sample.bin')))
+  })
+})