usenet-indexer/app/services/SevenZipService.ts

import { execFile } from 'node:child_process'
import { mkdtemp, open, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import NntpService from '#services/NntpService'
import { segmentsOf } from '#services/FileStore'
import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
import type { RarInfo } from '#lib/rar/index'
import {
  checkHeader,
  listingStart,
  parseBsdtarListing,
  parseSevenZipStart,
  sevenZipVolume,
  type SevenZipStart,
} from '#lib/sevenzip/index'

const run = promisify(execFile)

/** The tail read for a listing: header plus packed header; more is not a header. */
const MAX_TAIL_BYTES = 16 * 1024 * 1024

/**
 * Lists 7z archives, also split ones (.7z.001, .002, …), from the first
 * segment of the first volume and the last segment(s) of the last one
 * (CONCEPT.md D19). Those bytes are put at their place in a sparse file the
 * size of the whole archive, which bsdtar (libarchive) lists: it only reads
 * the start and the header at the end.
 */
class SevenZipService {
  /**
   * The volumes of the archive `file` belongs to, in order, when all of them
   * up to the highest number seen are there; null otherwise. An unsplit (or
   * unnamed) archive is its own only volume.
   */
  async volumes(file: any): Promise<any[] | null> {
    const name = String(file.real_name ?? file.filename)
    const volume = sevenZipVolume(name)
    if (!volume) return [file]
    if (!file.release_id) return null
    const base = name.replace(/\.\d+$/, '').toLowerCase()
    const rows = await db
      .from('files')
      .where('release_id', file.release_id)
      .whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
    const byNumber = new Map<number, any>()
    for (const row of rows) {
      const n = sevenZipVolume(String(row.real_name ?? row.filename))
      if (n && !byNumber.has(n)) byNumber.set(n, row)
    }
    const max = Math.max(0, ...byNumber.keys())
    const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
    return max > 0 && ordered.every(Boolean) ? ordered : null
  }

  /**
   * Lists the archive whose first volume starts with `firstData` (a decoded
   * first segment). Null when it isn't a 7z start, when volumes are missing,
   * or when the tail doesn't hold the header its start points to.
   */
  async list(firstFile: any, firstData: Buffer): Promise<RarInfo | null> {
    const start = parseSevenZipStart(firstData)
    if (!start) return null
    const volumes = await this.volumes(firstFile)
    if (!volumes) return null
    if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1)

    const tail = await this.readTail(volumes[volumes.length - 1], start)
    if (!tail) return null

    const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
    try {
      const archive = path.join(dir, 'archive.7z')
      const handle = await open(archive, 'w')
      try {
        await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
        await handle.write(tail.data, 0, tail.data.length, tail.from)
        await handle.truncate(start.archiveSize)
      } finally {
        await handle.close()
      }
      return await this.bsdtar(archive, volumes.length > 1)
    } finally {
      await rm(dir, { recursive: true, force: true })
    }
  }

  /** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
  private async bsdtar(archive: string, split: boolean): Promise<RarInfo | null> {
    let stdout = ''
    try {
      stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
    } catch (error: any) {
      if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split)
      logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
      return null
    }
    const entries = parseBsdtarListing(stdout)
    const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
    let encrypted = false
    if (sample) {
      try {
        await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
      } catch (error: any) {
        // Without the data (it's all zeros here), anything else fails too: only this message counts.
        encrypted = /content is encrypted/i.test(String(error.stderr))
      }
    }
    return this.info(entries, false, encrypted, split)
  }

  private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo {
    return {
      format: '7z',
      volume: split,
      firstVolume: true,
      solid: false,
      headersEncrypted,
      truncated: false,
      entries: entries.map((e) => ({
        name: e.name,
        size: e.size,
        packedSize: 0,
        directory: e.directory,
        encrypted: encrypted && !e.directory,
        splitBefore: false,
        splitAfter: false,
        stored: false,
        dataOffset: 0,
      })),
    }
  }

  /**
   * The archive's tail from the last volume: the header (checked against its
   * CRC) and, for an encoded header, the packed stream before it. `from` is
   * its position in the whole archive.
   */
  private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> {
    const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
    const numbers = [...ids.keys()]
    if (numbers.length === 0) return null
    const segments = new Map<number, DecodedSegment>()
    const segment = async (n: number) => {
      if (!segments.has(n)) {
        const id = ids.get(n)?.id
        if (!id) throw new Error(`part ${n} of ${last.filename} is missing`)
        segments.set(n, decodeSegment(await this.body(id), { lenientSize: true }))
      }
      return segments.get(n)!
    }

    const lastNumber = numbers[numbers.length - 1]
    const lastSegment = await segment(lastNumber)
    const volumeSize = lastSegment.meta.begin.size
    const volumeStart = start.archiveSize - volumeSize
    const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
    const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
    if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null

    // Bytes [from, to) of the whole archive, from the last volume's parts.
    const read = async (from: number, to: number): Promise<Buffer | null> => {
      if (from < volumeStart) return null
      const a = from - volumeStart
      const b = to - volumeStart
      const pieces: Buffer[] = []
      for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
        const s = await segment(n)
        const offset = (s.meta.part?.begin ?? 1) - 1
        if (offset !== (n - 1) * partSize) return null
        pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
      }
      const data = Buffer.concat(pieces)
      return data.length === b - a ? data : null
    }

    const header = await read(start.headerStart, start.archiveSize)
    if (!header || !checkHeader(start, header)) return null
    const from = Math.min(listingStart(start, header), start.headerStart)
    if (start.archiveSize - from > MAX_TAIL_BYTES) return null
    const data = await read(from, start.archiveSize)
    return data ? { from, data } : null
  }

  private async body(messageId: string): Promise<Buffer> {
    const conn = await NntpService.acquire()
    let failed = false
    try {
      return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
    } catch (error: any) {
      failed = !/^4\d\d/.test(String(error.message))
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
  }
}

export default new SevenZipService()