usenet-indexer/lib/sevenzip/parse.ts

import { crc32 } from 'node:zlib'
import { lzmaDecode } from '#lib/sevenzip/lzma'

/**
 * What a 7z archive's header says (its listing): read from the archive's end, where 7z keeps it, so the last
 * segment of the last volume is enough (the `7z-tail` probe; REBUILD.md, phase 2). The header there is mostly
 * an encoded one: a pointer to the real header, LZMA-packed just before it, or AES-encrypted (`7z -mhe`), which
 * hides the names and means a password.
 */
export interface SevenZipListing {
  /** Files, not directories, with their unpacked size (0 for an empty one). */
  files: { name: string; size: number }[]
  /** Entries or the header encrypted (AES). */
  encrypted: boolean
  /** The header itself encrypted: no names known. */
  headerEncrypted: boolean
}

const AES = '06f10701'
const LZMA = '030101'
/** A header bigger than this isn't unpacked (headers list names; 64 MiB is ~a million files). */
const MAX_HEADER = 64 * 1024 * 1024

const ID = {
  end: 0x00,
  header: 0x01,
  archiveProperties: 0x02,
  additionalStreamsInfo: 0x03,
  mainStreamsInfo: 0x04,
  filesInfo: 0x05,
  packInfo: 0x06,
  unpackInfo: 0x07,
  subStreamsInfo: 0x08,
  size: 0x09,
  crc: 0x0a,
  folder: 0x0b,
  codersUnpackSize: 0x0c,
  numUnpackStream: 0x0d,
  emptyStream: 0x0e,
  emptyFile: 0x0f,
  name: 0x11,
  encodedHeader: 0x17,
  dummy: 0x19,
}

class Malformed extends Error {}

interface Coder {
  id: string
  properties: Uint8Array
  inStreams: number
  outStreams: number
}
interface Folder {
  coders: Coder[]
  bindOut: Set<number>
  unpackSizes: number[]
  crc?: number
}
interface StreamsInfo {
  packPos: number
  packSizes: number[]
  folders: Folder[]
  /** Per folder, how many files' streams it holds, and every stream's size in order. */
  streamsPerFolder: number[]
  streamSizes: number[]
}

class Reader {
  constructor(
    private buf: Uint8Array,
    public pos = 0
  ) {}

  get done() {
    return this.pos >= this.buf.length
  }

  byte(): number {
    if (this.pos >= this.buf.length) throw new Malformed('7z: header ended')
    return this.buf[this.pos++]
  }

  bytes(n: number): Uint8Array {
    if (n < 0 || this.pos + n > this.buf.length) throw new Malformed('7z: header ended')
    const out = this.buf.subarray(this.pos, this.pos + n)
    this.pos += n
    return out
  }

  /** 7z's variable-length number: the first byte's high bits say how many bytes follow. */
  number(): number {
    const first = this.byte()
    let mask = 0x80
    let value = 0
    for (let i = 0; i < 8; i++) {
      if ((first & mask) === 0) {
        value += (first & (mask - 1)) * 2 ** (8 * i)
        break
      }
      value += this.byte() * 2 ** (8 * i)
      mask >>>= 1
    }
    if (value > Number.MAX_SAFE_INTEGER) throw new Malformed('7z: number too large')
    return value
  }

  uint32(): number {
    const b = this.bytes(4)
    return (b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24)) >>> 0
  }

  bits(n: number): boolean[] {
    const out: boolean[] = []
    let byte = 0
    for (let i = 0; i < n; i++) {
      if (i % 8 === 0) byte = this.byte()
      out.push((byte & (0x80 >>> i % 8)) !== 0)
    }
    return out
  }

  /** "All defined" or a bit vector, then a CRC per defined item. */
  digests(n: number): (number | undefined)[] {
    const allDefined = this.byte() !== 0
    const defined = allDefined ? Array(n).fill(true) : this.bits(n)
    return defined.map((d) => (d ? this.uint32() : undefined))
  }

  expect(id: number) {
    if (this.byte() !== id) throw new Malformed(`7z: expected property ${id}`)
  }
}

/** Counts that come from the data: a garbage position makes huge ones, which shouldn't allocate. */
function count(n: number, max = 1_000_000): number {
  if (n > max) throw new Malformed('7z: implausible count')
  return n
}

function readFolder(r: Reader): Folder {
  const coders: Coder[] = []
  const numCoders = count(r.number(), 64)
  if (numCoders === 0) throw new Malformed('7z: folder without coders')
  for (let i = 0; i < numCoders; i++) {
    const flags = r.byte()
    if (flags & 0x80) throw new Malformed('7z: alternative coder methods')
    const id = Buffer.from(r.bytes(flags & 0x0f)).toString('hex')
    const complex = (flags & 0x10) !== 0
    const inStreams = complex ? count(r.number(), 64) : 1
    const outStreams = complex ? count(r.number(), 64) : 1
    const properties = flags & 0x20 ? r.bytes(count(r.number(), 1 << 16)) : new Uint8Array(0)
    coders.push({ id, properties, inStreams, outStreams })
  }
  const totalOut = coders.reduce((n, c) => n + c.outStreams, 0)
  const totalIn = coders.reduce((n, c) => n + c.inStreams, 0)
  const bindOut = new Set<number>()
  for (let i = 0; i < totalOut - 1; i++) {
    r.number()
    bindOut.add(r.number())
  }
  const packed = totalIn - (totalOut - 1)
  if (packed < 1) throw new Malformed('7z: folder without packed streams')
  if (packed > 1) for (let i = 0; i < packed; i++) r.number()
  return { coders, bindOut, unpackSizes: [] }
}

/** The folder's output: the one out stream no bind pair consumes. */
function unpackSize(folder: Folder): number {
  for (let i = folder.unpackSizes.length - 1; i >= 0; i--) if (!folder.bindOut.has(i)) return folder.unpackSizes[i]
  return 0
}

function readStreamsInfo(r: Reader): StreamsInfo {
  const info: StreamsInfo = { packPos: 0, packSizes: [], folders: [], streamsPerFolder: [], streamSizes: [] }
  let id = r.byte()
  if (id === ID.packInfo) {
    info.packPos = r.number()
    const n = count(r.number())
    for (id = r.byte(); id !== ID.end; id = r.byte()) {
      if (id === ID.size) for (let i = 0; i < n; i++) info.packSizes.push(r.number())
      else if (id === ID.crc) r.digests(n)
      else throw new Malformed('7z: unexpected property in pack info')
    }
    id = r.byte()
  }
  if (id === ID.unpackInfo) {
    r.expect(ID.folder)
    const n = count(r.number())
    if (r.byte() !== 0) throw new Malformed('7z: external folders')
    for (let i = 0; i < n; i++) info.folders.push(readFolder(r))
    r.expect(ID.codersUnpackSize)
    for (const f of info.folders) for (const c of f.coders) for (let i = 0; i < c.outStreams; i++) f.unpackSizes.push(r.number())
    for (id = r.byte(); id !== ID.end; id = r.byte()) {
      if (id === ID.crc) r.digests(n).forEach((crc, i) => (info.folders[i].crc = crc))
      else throw new Malformed('7z: unexpected property in unpack info')
    }
    id = r.byte()
  }
  info.streamsPerFolder = info.folders.map(() => 1)
  if (id === ID.subStreamsInfo) {
    id = r.byte()
    if (id === ID.numUnpackStream) {
      info.streamsPerFolder = info.folders.map(() => count(r.number()))
      id = r.byte()
    }
    const hasSizes = id === ID.size
    info.folders.forEach((f, i) => {
      const n = info.streamsPerFolder[i]
      if (n === 0) return
      let sum = 0
      for (let j = 0; j < n - 1 && hasSizes; j++) {
        const size = r.number()
        info.streamSizes.push(size)
        sum += size
      }
      info.streamSizes.push(unpackSize(f) - sum)
    })
    if (hasSizes) id = r.byte()
    while (id !== ID.end) {
      if (id === ID.crc) {
        // CRCs for the streams whose folder had none (a folder of one stream with a CRC has its own).
        const unknown = info.folders.reduce((n, f, i) => n + (info.streamsPerFolder[i] === 1 && f.crc !== undefined ? 0 : info.streamsPerFolder[i]), 0)
        r.digests(unknown)
      } else {
        throw new Malformed('7z: unexpected property in substreams info')
      }
      id = r.byte()
    }
    id = r.byte()
  } else {
    info.streamSizes = info.folders.map(unpackSize)
  }
  if (id !== ID.end) throw new Malformed('7z: streams info not ended')
  return info
}

function readFilesInfo(r: Reader, streams: StreamsInfo | null): { name: string; size: number }[] {
  const n = count(r.number())
  let emptyStream: boolean[] = Array(n).fill(false)
  let emptyFile: boolean[] = []
  let names: string[] = []
  for (let id = r.byte(); id !== ID.end; id = r.byte()) {
    const size = r.number()
    const end = r.pos + size
    if (id === ID.emptyStream) {
      emptyStream = r.bits(n)
    } else if (id === ID.emptyFile) {
      emptyFile = r.bits(emptyStream.filter(Boolean).length)
    } else if (id === ID.name) {
      if (r.byte() !== 0) throw new Malformed('7z: external names')
      const text = Buffer.from(r.bytes(size - 1)).toString('utf16le')
      names = text.split('\0').slice(0, n)
    } else if (id !== ID.dummy && id > 0x19) {
      throw new Malformed('7z: unknown file property')
    }
    r.pos = end
  }
  if (names.length !== n) throw new Malformed('7z: names missing')
  const out: { name: string; size: number }[] = []
  let stream = 0
  let empty = 0
  for (let i = 0; i < n; i++) {
    if (!emptyStream[i]) {
      out.push({ name: names[i], size: streams?.streamSizes[stream++] ?? 0 })
      continue
    }
    // An empty stream is a directory unless it's marked an empty file.
    if (emptyFile[empty++]) out.push({ name: names[i], size: 0 })
  }
  return out
}

function readHeader(r: Reader): SevenZipListing {
  let id = r.byte()
  if (id === ID.archiveProperties) {
    for (let type = r.byte(); type !== 0; type = r.byte()) r.bytes(r.number())
    id = r.byte()
  }
  if (id === ID.additionalStreamsInfo) {
    readStreamsInfo(r)
    id = r.byte()
  }
  let streams: StreamsInfo | null = null
  if (id === ID.mainStreamsInfo) {
    streams = readStreamsInfo(r)
    id = r.byte()
  }
  let files: { name: string; size: number }[] = []
  if (id === ID.filesInfo) {
    files = readFilesInfo(r, streams)
    id = r.byte()
  }
  if (id !== ID.end) throw new Malformed('7z: header not ended')
  const encrypted = (streams?.folders ?? []).some((f) => f.coders.some((c) => c.id === AES))
  return { files, encrypted, headerEncrypted: false }
}

/**
 * The header that ends exactly at `end` in `data`, starting at `start`: a plain one, or an encoded one whose packed
 * data lies just before it (where 7z writes it). Throws when it isn't one.
 */
function headerAt(data: Uint8Array, start: number, end: number): SevenZipListing {
  const r = new Reader(data.subarray(0, end), start)
  const id = r.byte()
  if (id === ID.header) {
    const listing = readHeader(r)
    if (!r.done) throw new Malformed('7z: bytes after the header')
    return listing
  }
  if (id !== ID.encodedHeader) throw new Malformed('7z: not a header')
  const info = readStreamsInfo(r)
  if (!r.done) throw new Malformed('7z: bytes after the encoded header')
  if (info.folders.length !== 1 || info.packSizes.length < 1) throw new Malformed('7z: unexpected encoded header')
  const folder = info.folders[0]
  if (folder.coders.some((c) => c.id === AES)) return { files: [], encrypted: true, headerEncrypted: true }
  if (folder.coders.length !== 1 || folder.coders[0].id !== LZMA) throw new Malformed(`7z: header coder ${folder.coders[0].id} not supported`)
  const packSize = info.packSizes[0]
  const size = unpackSize(folder)
  if (packSize > start || size > MAX_HEADER) throw new Malformed('7z: packed header not in the data')
  const header = lzmaDecode(folder.coders[0].properties, data.subarray(start - packSize, start), size)
  if (folder.crc !== undefined && crc32(header) >>> 0 !== folder.crc) throw new Malformed('7z: header CRC mismatch')
  return headerAt(header, 0, header.length)
}

/**
 * The listing of the 7z archive that `data` is the end of (the last segment of its last volume, or a whole small
 * archive); null when no header ends there.
 */
export function parseSevenZipTail(data: Uint8Array): SevenZipListing | null {
  const end = data.length
  // An encoded header is a few dozen bytes; a plain one (7z -mhc=off) can be large.
  for (let start = end - 2; start >= Math.max(0, end - 1024); start--) {
    if (data[start] !== ID.encodedHeader || data[start + 1] !== ID.packInfo) continue
    try {
      return headerAt(data, start, end)
    } catch {
      // Not a header here (any byte can look like the start of one): the next position.
    }
  }
  for (let start = end - 2; start >= 0; start--) {
    if (data[start] !== ID.header || ![ID.archiveProperties, ID.additionalStreamsInfo, ID.mainStreamsInfo, ID.filesInfo, ID.end].includes(data[start + 1])) continue
    try {
      return headerAt(data, start, end)
    } catch {
      // Not a header here (any byte can look like the start of one): the next position.
    }
  }
  return null
}