Probe 7z listings and ZIP starts
7z keeps its file list at the end of the archive, so the 7z-tail candidates (the last segment of the highest volume) were queued but never probed, and the .7z.001 start only gave the type. The prober now claims 7z-tail candidates after the archives; lib/sevenzip finds the end header in the article, decodes the LZMA-packed header (a TypeScript port of the LZMA SDK's reference decoder) and lists the files with their sizes. An AES-encrypted header means passworded without names. Names feed the inner name and the judge's fake rule. An unsplit .7z of several segments now also gets a tail candidate, and .zip files an archive candidate: their first segment's local headers give the first entries and whether they are encrypted. Both in the Rust edge and the TypeScript indexer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
3b61215f6b
commit
d446bb57df
25 files changed
+4840
-11
No files matched your search
@@ -0,0 +1,350 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
import { lzmaDecode } from '#lib/sevenzip/lzma'
|
||||
|
||||
/**
|
||||
* What a 7z archive's header says (its listing): read from the archive's end, where 7z keeps it, so the last
|
||||
* segment of the last volume is enough (the `7z-tail` probe; REBUILD.md, phase 2). The header there is mostly
|
||||
* an encoded one: a pointer to the real header, LZMA-packed just before it, or AES-encrypted (`7z -mhe`), which
|
||||
* hides the names and means a password.
|
||||
*/
|
||||
export interface SevenZipListing {
|
||||
/** Files, not directories, with their unpacked size (0 for an empty one). */
|
||||
files: { name: string; size: number }[]
|
||||
/** Entries or the header encrypted (AES). */
|
||||
encrypted: boolean
|
||||
/** The header itself encrypted: no names known. */
|
||||
headerEncrypted: boolean
|
||||
}
|
||||
|
||||
const AES = '06f10701'
|
||||
const LZMA = '030101'
|
||||
/** A header bigger than this isn't unpacked (headers list names; 64 MiB is ~a million files). */
|
||||
const MAX_HEADER = 64 * 1024 * 1024
|
||||
|
||||
const ID = {
|
||||
end: 0x00,
|
||||
header: 0x01,
|
||||
archiveProperties: 0x02,
|
||||
additionalStreamsInfo: 0x03,
|
||||
mainStreamsInfo: 0x04,
|
||||
filesInfo: 0x05,
|
||||
packInfo: 0x06,
|
||||
unpackInfo: 0x07,
|
||||
subStreamsInfo: 0x08,
|
||||
size: 0x09,
|
||||
crc: 0x0a,
|
||||
folder: 0x0b,
|
||||
codersUnpackSize: 0x0c,
|
||||
numUnpackStream: 0x0d,
|
||||
emptyStream: 0x0e,
|
||||
emptyFile: 0x0f,
|
||||
name: 0x11,
|
||||
encodedHeader: 0x17,
|
||||
dummy: 0x19,
|
||||
}
|
||||
|
||||
class Malformed extends Error {}
|
||||
|
||||
interface Coder {
|
||||
id: string
|
||||
properties: Uint8Array
|
||||
inStreams: number
|
||||
outStreams: number
|
||||
}
|
||||
interface Folder {
|
||||
coders: Coder[]
|
||||
bindOut: Set<number>
|
||||
unpackSizes: number[]
|
||||
crc?: number
|
||||
}
|
||||
interface StreamsInfo {
|
||||
packPos: number
|
||||
packSizes: number[]
|
||||
folders: Folder[]
|
||||
/** Per folder, how many files' streams it holds, and every stream's size in order. */
|
||||
streamsPerFolder: number[]
|
||||
streamSizes: number[]
|
||||
}
|
||||
|
||||
class Reader {
|
||||
constructor(
|
||||
private buf: Uint8Array,
|
||||
public pos = 0
|
||||
) {}
|
||||
|
||||
get done() {
|
||||
return this.pos >= this.buf.length
|
||||
}
|
||||
|
||||
byte(): number {
|
||||
if (this.pos >= this.buf.length) throw new Malformed('7z: header ended')
|
||||
return this.buf[this.pos++]
|
||||
}
|
||||
|
||||
bytes(n: number): Uint8Array {
|
||||
if (n < 0 || this.pos + n > this.buf.length) throw new Malformed('7z: header ended')
|
||||
const out = this.buf.subarray(this.pos, this.pos + n)
|
||||
this.pos += n
|
||||
return out
|
||||
}
|
||||
|
||||
/** 7z's variable-length number: the first byte's high bits say how many bytes follow. */
|
||||
number(): number {
|
||||
const first = this.byte()
|
||||
let mask = 0x80
|
||||
let value = 0
|
||||
for (let i = 0; i < 8; i++) {
|
||||
if ((first & mask) === 0) {
|
||||
value += (first & (mask - 1)) * 2 ** (8 * i)
|
||||
break
|
||||
}
|
||||
value += this.byte() * 2 ** (8 * i)
|
||||
mask >>>= 1
|
||||
}
|
||||
if (value > Number.MAX_SAFE_INTEGER) throw new Malformed('7z: number too large')
|
||||
return value
|
||||
}
|
||||
|
||||
uint32(): number {
|
||||
const b = this.bytes(4)
|
||||
return (b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24)) >>> 0
|
||||
}
|
||||
|
||||
bits(n: number): boolean[] {
|
||||
const out: boolean[] = []
|
||||
let byte = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (i % 8 === 0) byte = this.byte()
|
||||
out.push((byte & (0x80 >>> i % 8)) !== 0)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** "All defined" or a bit vector, then a CRC per defined item. */
|
||||
digests(n: number): (number | undefined)[] {
|
||||
const allDefined = this.byte() !== 0
|
||||
const defined = allDefined ? Array(n).fill(true) : this.bits(n)
|
||||
return defined.map((d) => (d ? this.uint32() : undefined))
|
||||
}
|
||||
|
||||
expect(id: number) {
|
||||
if (this.byte() !== id) throw new Malformed(`7z: expected property ${id}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Counts that come from the data: a garbage position makes huge ones, which shouldn't allocate. */
|
||||
function count(n: number, max = 1_000_000): number {
|
||||
if (n > max) throw new Malformed('7z: implausible count')
|
||||
return n
|
||||
}
|
||||
|
||||
function readFolder(r: Reader): Folder {
|
||||
const coders: Coder[] = []
|
||||
const numCoders = count(r.number(), 64)
|
||||
if (numCoders === 0) throw new Malformed('7z: folder without coders')
|
||||
for (let i = 0; i < numCoders; i++) {
|
||||
const flags = r.byte()
|
||||
if (flags & 0x80) throw new Malformed('7z: alternative coder methods')
|
||||
const id = Buffer.from(r.bytes(flags & 0x0f)).toString('hex')
|
||||
const complex = (flags & 0x10) !== 0
|
||||
const inStreams = complex ? count(r.number(), 64) : 1
|
||||
const outStreams = complex ? count(r.number(), 64) : 1
|
||||
const properties = flags & 0x20 ? r.bytes(count(r.number(), 1 << 16)) : new Uint8Array(0)
|
||||
coders.push({ id, properties, inStreams, outStreams })
|
||||
}
|
||||
const totalOut = coders.reduce((n, c) => n + c.outStreams, 0)
|
||||
const totalIn = coders.reduce((n, c) => n + c.inStreams, 0)
|
||||
const bindOut = new Set<number>()
|
||||
for (let i = 0; i < totalOut - 1; i++) {
|
||||
r.number()
|
||||
bindOut.add(r.number())
|
||||
}
|
||||
const packed = totalIn - (totalOut - 1)
|
||||
if (packed < 1) throw new Malformed('7z: folder without packed streams')
|
||||
if (packed > 1) for (let i = 0; i < packed; i++) r.number()
|
||||
return { coders, bindOut, unpackSizes: [] }
|
||||
}
|
||||
|
||||
/** The folder's output: the one out stream no bind pair consumes. */
|
||||
function unpackSize(folder: Folder): number {
|
||||
for (let i = folder.unpackSizes.length - 1; i >= 0; i--) if (!folder.bindOut.has(i)) return folder.unpackSizes[i]
|
||||
return 0
|
||||
}
|
||||
|
||||
function readStreamsInfo(r: Reader): StreamsInfo {
|
||||
const info: StreamsInfo = { packPos: 0, packSizes: [], folders: [], streamsPerFolder: [], streamSizes: [] }
|
||||
let id = r.byte()
|
||||
if (id === ID.packInfo) {
|
||||
info.packPos = r.number()
|
||||
const n = count(r.number())
|
||||
for (id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
if (id === ID.size) for (let i = 0; i < n; i++) info.packSizes.push(r.number())
|
||||
else if (id === ID.crc) r.digests(n)
|
||||
else throw new Malformed('7z: unexpected property in pack info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
if (id === ID.unpackInfo) {
|
||||
r.expect(ID.folder)
|
||||
const n = count(r.number())
|
||||
if (r.byte() !== 0) throw new Malformed('7z: external folders')
|
||||
for (let i = 0; i < n; i++) info.folders.push(readFolder(r))
|
||||
r.expect(ID.codersUnpackSize)
|
||||
for (const f of info.folders) for (const c of f.coders) for (let i = 0; i < c.outStreams; i++) f.unpackSizes.push(r.number())
|
||||
for (id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
if (id === ID.crc) r.digests(n).forEach((crc, i) => (info.folders[i].crc = crc))
|
||||
else throw new Malformed('7z: unexpected property in unpack info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
info.streamsPerFolder = info.folders.map(() => 1)
|
||||
if (id === ID.subStreamsInfo) {
|
||||
id = r.byte()
|
||||
if (id === ID.numUnpackStream) {
|
||||
info.streamsPerFolder = info.folders.map(() => count(r.number()))
|
||||
id = r.byte()
|
||||
}
|
||||
const hasSizes = id === ID.size
|
||||
info.folders.forEach((f, i) => {
|
||||
const n = info.streamsPerFolder[i]
|
||||
if (n === 0) return
|
||||
let sum = 0
|
||||
for (let j = 0; j < n - 1 && hasSizes; j++) {
|
||||
const size = r.number()
|
||||
info.streamSizes.push(size)
|
||||
sum += size
|
||||
}
|
||||
info.streamSizes.push(unpackSize(f) - sum)
|
||||
})
|
||||
if (hasSizes) id = r.byte()
|
||||
while (id !== ID.end) {
|
||||
if (id === ID.crc) {
|
||||
// CRCs for the streams whose folder had none (a folder of one stream with a CRC has its own).
|
||||
const unknown = info.folders.reduce((n, f, i) => n + (info.streamsPerFolder[i] === 1 && f.crc !== undefined ? 0 : info.streamsPerFolder[i]), 0)
|
||||
r.digests(unknown)
|
||||
} else {
|
||||
throw new Malformed('7z: unexpected property in substreams info')
|
||||
}
|
||||
id = r.byte()
|
||||
}
|
||||
id = r.byte()
|
||||
} else {
|
||||
info.streamSizes = info.folders.map(unpackSize)
|
||||
}
|
||||
if (id !== ID.end) throw new Malformed('7z: streams info not ended')
|
||||
return info
|
||||
}
|
||||
|
||||
function readFilesInfo(r: Reader, streams: StreamsInfo | null): { name: string; size: number }[] {
|
||||
const n = count(r.number())
|
||||
let emptyStream: boolean[] = Array(n).fill(false)
|
||||
let emptyFile: boolean[] = []
|
||||
let names: string[] = []
|
||||
for (let id = r.byte(); id !== ID.end; id = r.byte()) {
|
||||
const size = r.number()
|
||||
const end = r.pos + size
|
||||
if (id === ID.emptyStream) {
|
||||
emptyStream = r.bits(n)
|
||||
} else if (id === ID.emptyFile) {
|
||||
emptyFile = r.bits(emptyStream.filter(Boolean).length)
|
||||
} else if (id === ID.name) {
|
||||
if (r.byte() !== 0) throw new Malformed('7z: external names')
|
||||
const text = Buffer.from(r.bytes(size - 1)).toString('utf16le')
|
||||
names = text.split('\0').slice(0, n)
|
||||
} else if (id !== ID.dummy && id > 0x19) {
|
||||
throw new Malformed('7z: unknown file property')
|
||||
}
|
||||
r.pos = end
|
||||
}
|
||||
if (names.length !== n) throw new Malformed('7z: names missing')
|
||||
const out: { name: string; size: number }[] = []
|
||||
let stream = 0
|
||||
let empty = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (!emptyStream[i]) {
|
||||
out.push({ name: names[i], size: streams?.streamSizes[stream++] ?? 0 })
|
||||
continue
|
||||
}
|
||||
// An empty stream is a directory unless it's marked an empty file.
|
||||
if (emptyFile[empty++]) out.push({ name: names[i], size: 0 })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function readHeader(r: Reader): SevenZipListing {
|
||||
let id = r.byte()
|
||||
if (id === ID.archiveProperties) {
|
||||
for (let type = r.byte(); type !== 0; type = r.byte()) r.bytes(r.number())
|
||||
id = r.byte()
|
||||
}
|
||||
if (id === ID.additionalStreamsInfo) {
|
||||
readStreamsInfo(r)
|
||||
id = r.byte()
|
||||
}
|
||||
let streams: StreamsInfo | null = null
|
||||
if (id === ID.mainStreamsInfo) {
|
||||
streams = readStreamsInfo(r)
|
||||
id = r.byte()
|
||||
}
|
||||
let files: { name: string; size: number }[] = []
|
||||
if (id === ID.filesInfo) {
|
||||
files = readFilesInfo(r, streams)
|
||||
id = r.byte()
|
||||
}
|
||||
if (id !== ID.end) throw new Malformed('7z: header not ended')
|
||||
const encrypted = (streams?.folders ?? []).some((f) => f.coders.some((c) => c.id === AES))
|
||||
return { files, encrypted, headerEncrypted: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* The header that ends exactly at `end` in `data`, starting at `start`: a plain one, or an encoded one whose packed
|
||||
* data lies just before it (where 7z writes it). Throws when it isn't one.
|
||||
*/
|
||||
function headerAt(data: Uint8Array, start: number, end: number): SevenZipListing {
|
||||
const r = new Reader(data.subarray(0, end), start)
|
||||
const id = r.byte()
|
||||
if (id === ID.header) {
|
||||
const listing = readHeader(r)
|
||||
if (!r.done) throw new Malformed('7z: bytes after the header')
|
||||
return listing
|
||||
}
|
||||
if (id !== ID.encodedHeader) throw new Malformed('7z: not a header')
|
||||
const info = readStreamsInfo(r)
|
||||
if (!r.done) throw new Malformed('7z: bytes after the encoded header')
|
||||
if (info.folders.length !== 1 || info.packSizes.length < 1) throw new Malformed('7z: unexpected encoded header')
|
||||
const folder = info.folders[0]
|
||||
if (folder.coders.some((c) => c.id === AES)) return { files: [], encrypted: true, headerEncrypted: true }
|
||||
if (folder.coders.length !== 1 || folder.coders[0].id !== LZMA) throw new Malformed(`7z: header coder ${folder.coders[0].id} not supported`)
|
||||
const packSize = info.packSizes[0]
|
||||
const size = unpackSize(folder)
|
||||
if (packSize > start || size > MAX_HEADER) throw new Malformed('7z: packed header not in the data')
|
||||
const header = lzmaDecode(folder.coders[0].properties, data.subarray(start - packSize, start), size)
|
||||
if (folder.crc !== undefined && crc32(header) >>> 0 !== folder.crc) throw new Malformed('7z: header CRC mismatch')
|
||||
return headerAt(header, 0, header.length)
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing of the 7z archive that `data` is the end of (the last segment of its last volume, or a whole small
|
||||
* archive); null when no header ends there.
|
||||
*/
|
||||
export function parseSevenZipTail(data: Uint8Array): SevenZipListing | null {
|
||||
const end = data.length
|
||||
// An encoded header is a few dozen bytes; a plain one (7z -mhc=off) can be large.
|
||||
for (let start = end - 2; start >= Math.max(0, end - 1024); start--) {
|
||||
if (data[start] !== ID.encodedHeader || data[start + 1] !== ID.packInfo) continue
|
||||
try {
|
||||
return headerAt(data, start, end)
|
||||
} catch {
|
||||
// Not a header here (any byte can look like the start of one): the next position.
|
||||
}
|
||||
}
|
||||
for (let start = end - 2; start >= 0; start--) {
|
||||
if (data[start] !== ID.header || ![ID.archiveProperties, ID.additionalStreamsInfo, ID.mainStreamsInfo, ID.filesInfo, ID.end].includes(data[start + 1])) continue
|
||||
try {
|
||||
return headerAt(data, start, end)
|
||||
} catch {
|
||||
// Not a header here (any byte can look like the start of one): the next position.
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
Reference in new issue
Block a user