7z: encrypted headers are known from the encoded header, without bsdtar

85% of the 7z archives inspected have encrypted headers (-mhe). Their
encoded header lists the 7zAES coder, so the packed header stream is no
longer downloaded and bsdtar no longer runs for them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/SevenZipService.ts

@@ -11,6 +11,7 @@ import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
 import type { RarInfo } from '#lib/rar/index'
 import {
   checkHeader,
+  encodedHeaderEncrypted,
   listingStart,
   parseBsdtarListing,
   parseSevenZipStart,
@@ -70,6 +71,8 @@ class SevenZipService {
 
     const tail = await this.readTail(volumes[volumes.length - 1], start)
     if (!tail) return null
+    // Encrypted headers (-mhe): known from the header alone, no listing possible (most 7z uploads).
+    if ('headersEncrypted' in tail) return this.info([], true, true, volumes.length > 1)
 
     const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
     try {
@@ -140,7 +143,7 @@ class SevenZipService {
    * CRC) and, for an encoded header, the packed stream before it. `from` is
    * its position in the whole archive.
    */
-  private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> {
+  private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | { headersEncrypted: true } | null> {
     const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
     const numbers = [...ids.keys()]
     if (numbers.length === 0) return null
@@ -180,6 +183,7 @@ class SevenZipService {
 
     const header = await read(start.headerStart, start.archiveSize)
     if (!header || !checkHeader(start, header)) return null
+    if (encodedHeaderEncrypted(header)) return { headersEncrypted: true }
     const from = Math.min(listingStart(start, header), start.headerStart)
     if (start.archiveSize - from > MAX_TAIL_BYTES) return null
     const data = await read(from, start.archiveSize)

lib/sevenzip/index.ts

@@ -12,8 +12,13 @@ import { crc32 } from 'node:zlib'
 const SIGNATURE = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])
 const SIGNATURE_HEADER_SIZE = 32
 /** Header property ids (7zFormat.txt). */
-const K_ENCODED_HEADER = 0x17
+const K_END = 0x00
 const K_PACK_INFO = 0x06
+const K_UNPACK_INFO = 0x07
+const K_SIZE = 0x09
+const K_CRC = 0x0a
+const K_FOLDER = 0x0b
+const K_ENCODED_HEADER = 0x17
 
 export interface SevenZipStart {
   /** Where the archive header starts, from the start of the archive. */
@@ -49,6 +54,68 @@ export function listingStart(start: SevenZipStart, header: Buffer): number {
   return SIGNATURE_HEADER_SIZE + packPos
 }
 
+/** 7zAES, the coder of password-protected streams. */
+const AES_CODER = Buffer.from([0x06, 0xf1, 0x07, 0x01])
+
+/**
+ * Whether an encoded header (`header`, the bytes at headerStart) is
+ * encrypted (7z -mhe): its coders include 7zAES. Then nothing can be listed
+ * without the password, and neither the packed header nor bsdtar is needed.
+ * Null when the header isn't encoded or doesn't read as expected.
+ */
+export function encodedHeaderEncrypted(header: Buffer): boolean | null {
+  try {
+    if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return null
+    let pos = 2
+    let numPackStreams: number
+    ;[, pos] = readNumber(header, pos) // packPos
+    ;[numPackStreams, pos] = readNumber(header, pos)
+    for (let id = header[pos++]; id !== K_END; id = header[pos++]) {
+      if (id === K_SIZE) {
+        for (let i = 0; i < numPackStreams; i++) [, pos] = readNumber(header, pos)
+      } else if (id === K_CRC) {
+        const allDefined = header[pos++]
+        let defined = numPackStreams
+        if (!allDefined) {
+          const bits = header.subarray(pos, pos + Math.ceil(numPackStreams / 8))
+          pos += bits.length
+          defined = [...bits].reduce((sum, byte) => sum + byte.toString(2).replaceAll('0', '').length, 0)
+        }
+        pos += 4 * defined
+      } else {
+        return null
+      }
+      if (pos >= header.length) return null
+    }
+    if (header[pos++] !== K_UNPACK_INFO || header[pos++] !== K_FOLDER) return null
+    let numFolders: number
+    ;[numFolders, pos] = readNumber(header, pos)
+    if (numFolders < 1 || header[pos++] !== 0) return null // folders stored elsewhere
+    let numCoders: number
+    ;[numCoders, pos] = readNumber(header, pos)
+    if (!(numCoders >= 1) || pos >= header.length) return null
+    for (let c = 0; c < numCoders; c++) {
+      const flags = header[pos++]
+      const idSize = flags & 0x0f
+      if (header.subarray(pos, pos + idSize).equals(AES_CODER)) return true
+      pos += idSize
+      if (flags & 0x10) {
+        ;[, pos] = readNumber(header, pos)
+        ;[, pos] = readNumber(header, pos)
+      }
+      if (flags & 0x20) {
+        let size: number
+        ;[size, pos] = readNumber(header, pos)
+        pos += size
+      }
+      if (pos > header.length) return null
+    }
+    return false
+  } catch {
+    return null
+  }
+}
+
 /**
  * 7z's variable-length number: the leading 1-bits of the first byte count
  * the little-endian bytes that follow; its remaining bits are the high part.

tests/unit/sevenzip.spec.ts

@@ -1,6 +1,7 @@
 import { test } from '@japa/runner'
 import {
   checkHeader,
+  encodedHeaderEncrypted,
   listingStart,
   parseBsdtarListing,
   parseSevenZipStart,
@@ -19,7 +20,30 @@ const PLAIN = Buffer.from(
   'base64'
 )
 
+// `7z a -pSecret -mhe=on`: an encrypted header; with -mhe=off only the file data is encrypted.
+const HEADER_ENCRYPTED = Buffer.from(
+  'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
+  'base64'
+)
+const DATA_ENCRYPTED = Buffer.from(
+  'N3q8ryccAASBXGg3kQAAAAAAAAAiAAAAAAAAAO4xlWU16N8rTbvoF1dbq3eLPdZiAACBMweuD87ysgwHsMPa919FiphISvi++OI2giqcUPCAdmXzZtkqFSqP0Nr4rWaRH5dR+vNA2jJGIbDAZpbQM/ka9oh27tkqKb/+dHyx1SFHkxbLyWvd7W0qwaDHxSvW2fTpRdJSmi74crF957Qwy+P8dDF0MOYr5AGHAQHJSAAAFwYQAQmAgQAHCwEAASMDAQEFXQAQAAAMgKYKAaUZQIYAAA==',
+  'base64'
+)
+const headerOf = (archive: Buffer) => {
+  const start = parseSevenZipStart(archive)!
+  return archive.subarray(start.headerStart, start.archiveSize)
+}
+
 test.group('sevenzip', () => {
+  test('tells encrypted headers from the encoded header alone', ({ assert }) => {
+    assert.isTrue(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED)))
+    assert.isFalse(encodedHeaderEncrypted(headerOf(DATA_ENCRYPTED)))
+    assert.isFalse(encodedHeaderEncrypted(headerOf(ENCODED)))
+    // A plain header isn't encoded: bsdtar lists it.
+    assert.isNull(encodedHeaderEncrypted(headerOf(PLAIN)))
+    assert.isNull(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED).subarray(0, 12)))
+  })
+
   test('finds the header at the end from the signature header', ({ assert }) => {
     const start = parseSevenZipStart(ENCODED)!
     assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })