Daan MeijerandClaude Opus 5.5
f4ded062df
Inspection lists only the entries whose headers are in the first segment of a first volume. A walk hops from header to header, fetching one segment per entry and volume, in its own queue (one job per collection worker) on NNTP connections inspection isn't waiting for. content:walk-archives queues releases inspected earlier (CONCEPT D40). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
352 lines
11 KiB
TypeScript
352 lines
11 KiB
TypeScript
import { crc32 } from 'node:zlib'
|
||
|
||
export interface RarEntry {
|
||
name: string
|
||
/** Unpacked size in bytes (null when the archiver did not record it). */
|
||
size: number | null
|
||
packedSize: number
|
||
directory: boolean
|
||
/** The file's data is password protected. */
|
||
encrypted: boolean
|
||
/** Continues from the previous volume. */
|
||
splitBefore: boolean
|
||
/** Continues in the next volume. */
|
||
splitAfter: boolean
|
||
/** Stored without compression, so the data can be copied out as is. */
|
||
stored: boolean
|
||
/** Offset of the entry's (packed) data in the parsed buffer. */
|
||
dataOffset: number
|
||
}
|
||
|
||
export interface RarInfo {
|
||
/** 7z listings take the same shape (see lib/sevenzip). */
|
||
format: 'rar4' | 'rar5' | '7z'
|
||
/** Part of a multi-volume set. */
|
||
volume: boolean
|
||
/** First volume of the set (null when the format version does not say). */
|
||
firstVolume: boolean | null
|
||
solid: boolean
|
||
/** File names are encrypted too; nothing can be listed without the password. */
|
||
headersEncrypted: boolean
|
||
entries: RarEntry[]
|
||
/** The buffer ended before the end-of-archive header (normal for one segment). */
|
||
truncated: boolean
|
||
/**
|
||
* Offset of the first block that was not read: where the next header
|
||
* starts (possibly beyond the buffer, after an entry's data), or the
|
||
* header the buffer cut off. After the end-of-archive header, its end.
|
||
*/
|
||
next: number
|
||
}
|
||
|
||
const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
|
||
const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
||
|
||
export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
|
||
if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5'
|
||
if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4'
|
||
return null
|
||
}
|
||
|
||
/**
|
||
* Lists a RAR archive from its headers only, so the first segment of a first
|
||
* volume is enough. Returns null when the data is not a RAR archive.
|
||
*
|
||
* With `continueAs`, `data` starts at a header in the middle of a volume (no
|
||
* signature), e.g. where a previous parse said the next header is.
|
||
*/
|
||
export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
|
||
if (continueAs === 'rar4') return parseRar4(data, 0)
|
||
if (continueAs === 'rar5') return parseRar5(data, 0)
|
||
const format = rarFormat(data)
|
||
if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
|
||
if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
|
||
return null
|
||
}
|
||
|
||
// --- RAR 4.x (RAR 1.5 – 4.x archive format) -------------------------------
|
||
|
||
const R4_MAIN = 0x73
|
||
const R4_FILE = 0x74
|
||
const R4_END = 0x7b
|
||
const R4_LONG_BLOCK = 0x8000
|
||
|
||
function parseRar4(data: Buffer, start: number): RarInfo {
|
||
const info: RarInfo = {
|
||
format: 'rar4',
|
||
volume: false,
|
||
firstVolume: null,
|
||
solid: false,
|
||
headersEncrypted: false,
|
||
entries: [],
|
||
truncated: true,
|
||
next: start,
|
||
}
|
||
let p = start
|
||
|
||
while (p + 7 <= data.length) {
|
||
const headCrc = data.readUInt16LE(p)
|
||
const type = data[p + 2]
|
||
const flags = data.readUInt16LE(p + 3)
|
||
const size = data.readUInt16LE(p + 5)
|
||
if (size < 7 || p + size > data.length) break
|
||
|
||
if (type === R4_END) {
|
||
info.truncated = false
|
||
p += size
|
||
break
|
||
}
|
||
|
||
let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0)
|
||
|
||
if (type === R4_MAIN) {
|
||
info.volume = Boolean(flags & 0x0001)
|
||
info.solid = Boolean(flags & 0x0008)
|
||
info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true
|
||
if (flags & 0x0080) {
|
||
info.headersEncrypted = true
|
||
break
|
||
}
|
||
} else if (type === R4_FILE) {
|
||
if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break
|
||
const large = Boolean(flags & 0x0100)
|
||
let packed = data.readUInt32LE(p + 7)
|
||
let unpacked = data.readUInt32LE(p + 11)
|
||
const nameSize = data.readUInt16LE(p + 26)
|
||
let nameStart = p + 32
|
||
if (large) {
|
||
packed += data.readUInt32LE(p + 32) * 2 ** 32
|
||
unpacked += data.readUInt32LE(p + 36) * 2 ** 32
|
||
nameStart = p + 40
|
||
}
|
||
const rawName = data.subarray(nameStart, nameStart + nameSize)
|
||
info.entries.push({
|
||
name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'),
|
||
// 0xffffffff with LHD_LARGE unset means "unknown" in some archivers.
|
||
size: unpacked === 0xffffffff && !large ? null : unpacked,
|
||
packedSize: packed,
|
||
directory: (flags & 0x00e0) === 0x00e0,
|
||
encrypted: Boolean(flags & 0x0004),
|
||
splitBefore: Boolean(flags & 0x0001),
|
||
splitAfter: Boolean(flags & 0x0002),
|
||
stored: data[p + 25] === 0x30,
|
||
dataOffset: p + size,
|
||
})
|
||
next = p + size + packed
|
||
}
|
||
|
||
if (next <= p) break
|
||
p = next
|
||
}
|
||
// Archivers before RAR 3.0 don't set the first-volume flag; a first
|
||
// volume is one whose first file does not continue from a previous one.
|
||
if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
|
||
info.firstVolume = true
|
||
}
|
||
info.next = p
|
||
return info
|
||
}
|
||
|
||
/**
|
||
* RAR 4 stores Unicode names as "ASCII\0<compressed UTF-16>", where the
|
||
* compressed part refers back to the ASCII bytes (unrar's DecodeFileName).
|
||
*/
|
||
export function decodeRar4Name(raw: Buffer): string {
|
||
const zero = raw.indexOf(0)
|
||
if (zero === -1) return raw.toString('utf8')
|
||
const ascii = raw.subarray(0, zero)
|
||
const enc = raw.subarray(zero + 1)
|
||
if (enc.length === 0) return ascii.toString('latin1')
|
||
|
||
const out: number[] = []
|
||
const highByte = enc[0]
|
||
let encPos = 1
|
||
let flags = 0
|
||
let flagBits = 0
|
||
while (encPos < enc.length) {
|
||
if (flagBits === 0) {
|
||
flags = enc[encPos++]
|
||
flagBits = 8
|
||
}
|
||
switch (flags >> 6) {
|
||
case 0:
|
||
out.push(enc[encPos++])
|
||
break
|
||
case 1:
|
||
out.push(enc[encPos++] + (highByte << 8))
|
||
break
|
||
case 2:
|
||
out.push(enc[encPos] + (enc[encPos + 1] << 8))
|
||
encPos += 2
|
||
break
|
||
case 3: {
|
||
let length = enc[encPos++]
|
||
if (length & 0x80) {
|
||
const correction = enc[encPos++]
|
||
for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) {
|
||
out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8))
|
||
}
|
||
} else {
|
||
for (length += 2; length > 0 && out.length < ascii.length; length--) {
|
||
out.push(ascii[out.length])
|
||
}
|
||
}
|
||
break
|
||
}
|
||
}
|
||
flags = (flags << 2) & 0xff
|
||
flagBits -= 2
|
||
}
|
||
return String.fromCharCode(...out)
|
||
}
|
||
|
||
// --- RAR 5.0 ----------------------------------------------------------------
|
||
|
||
const R5_MAIN = 1
|
||
const R5_FILE = 2
|
||
const R5_ENCRYPTION = 4
|
||
const R5_END = 5
|
||
|
||
/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */
|
||
function vint(data: Buffer, offset: number): { value: number; next: number } | null {
|
||
let value = 0
|
||
let shift = 1
|
||
for (let i = offset; i < data.length && i < offset + 10; i++) {
|
||
value += (data[i] & 0x7f) * shift
|
||
shift *= 128
|
||
if (!(data[i] & 0x80)) return { value, next: i + 1 }
|
||
}
|
||
return null
|
||
}
|
||
|
||
function parseRar5(data: Buffer, start: number): RarInfo {
|
||
const info: RarInfo = {
|
||
format: 'rar5',
|
||
volume: false,
|
||
firstVolume: null,
|
||
solid: false,
|
||
headersEncrypted: false,
|
||
entries: [],
|
||
truncated: true,
|
||
next: start,
|
||
}
|
||
let p = start
|
||
|
||
while (p + 5 <= data.length) {
|
||
const headCrc = data.readUInt32LE(p)
|
||
const headerSize = vint(data, p + 4)
|
||
if (!headerSize || headerSize.value === 0) break
|
||
const headerEnd = headerSize.next + headerSize.value
|
||
if (headerEnd > data.length) break
|
||
if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break
|
||
|
||
const type = vint(data, headerSize.next)
|
||
const flags = type && vint(data, type.next)
|
||
if (!type || !flags) break
|
||
let cursor = flags.next
|
||
let extraSize = 0
|
||
let dataSize = 0
|
||
if (flags.value & 0x0001) {
|
||
const v = vint(data, cursor)
|
||
if (!v) break
|
||
extraSize = v.value
|
||
cursor = v.next
|
||
}
|
||
if (flags.value & 0x0002) {
|
||
const v = vint(data, cursor)
|
||
if (!v) break
|
||
dataSize = v.value
|
||
cursor = v.next
|
||
}
|
||
|
||
if (type.value === R5_END) {
|
||
info.truncated = false
|
||
p = headerEnd
|
||
break
|
||
}
|
||
if (type.value === R5_ENCRYPTION) {
|
||
info.headersEncrypted = true
|
||
break
|
||
}
|
||
|
||
if (type.value === R5_MAIN) {
|
||
const archiveFlags = vint(data, cursor)
|
||
if (!archiveFlags) break
|
||
info.volume = Boolean(archiveFlags.value & 0x0001)
|
||
// A volume number is only stored for volumes after the first.
|
||
info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true
|
||
info.solid = Boolean(archiveFlags.value & 0x0004)
|
||
} else if (type.value === R5_FILE) {
|
||
const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize)
|
||
if (!entry) break
|
||
info.entries.push(entry)
|
||
}
|
||
|
||
const next = headerEnd + dataSize
|
||
if (next <= p) break
|
||
p = next
|
||
}
|
||
info.next = p
|
||
return info
|
||
}
|
||
|
||
function parseRar5File(
|
||
data: Buffer,
|
||
start: number,
|
||
extraStart: number,
|
||
headerEnd: number,
|
||
headerFlags: number,
|
||
dataSize: number
|
||
): RarEntry | null {
|
||
const fileFlags = vint(data, start)
|
||
const unpacked = fileFlags && vint(data, fileFlags.next)
|
||
const attributes = unpacked && vint(data, unpacked.next)
|
||
if (!fileFlags || !unpacked || !attributes) return null
|
||
let cursor = attributes.next
|
||
if (fileFlags.value & 0x0002) cursor += 4 // mtime
|
||
if (fileFlags.value & 0x0004) cursor += 4 // data CRC32
|
||
const compression = vint(data, cursor)
|
||
const hostOs = compression && vint(data, compression.next)
|
||
const nameLength = hostOs && vint(data, hostOs.next)
|
||
if (!compression || !hostOs || !nameLength) return null
|
||
const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8')
|
||
// Compression information: bits 7-9 are the method, 0 is "store".
|
||
const method = Math.floor(compression.value / 128) & 0x7
|
||
|
||
// Extra area records: size, type, data. Type 1 is file encryption.
|
||
let encrypted = false
|
||
let e = extraStart
|
||
while (e < headerEnd) {
|
||
const recordSize = vint(data, e)
|
||
const recordType = recordSize && vint(data, recordSize.next)
|
||
if (!recordSize || !recordType || recordSize.value === 0) break
|
||
if (recordType.value === 0x01) encrypted = true
|
||
e = recordSize.next + recordSize.value
|
||
}
|
||
|
||
return {
|
||
name,
|
||
size: fileFlags.value & 0x0008 ? null : unpacked.value,
|
||
packedSize: dataSize,
|
||
directory: Boolean(fileFlags.value & 0x0001),
|
||
encrypted,
|
||
splitBefore: Boolean(headerFlags & 0x0008),
|
||
splitAfter: Boolean(headerFlags & 0x0010),
|
||
stored: method === 0,
|
||
dataOffset: headerEnd,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* The data of an entry stored without compression, copied out of the
|
||
* archive buffer. Null when it is compressed, encrypted, split over volumes
|
||
* or not completely in the buffer.
|
||
*/
|
||
export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null {
|
||
if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
||
if (entry.size === null || entry.size !== entry.packedSize) return null
|
||
const end = entry.dataOffset + entry.packedSize
|
||
if (end > data.length) return null
|
||
return data.subarray(entry.dataOffset, end)
|
||
}
|