import { execFile } from 'node:child_process' import { mkdtemp, open, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { promisify } from 'node:util' import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import NntpService from '#services/NntpService' import { segmentsOf } from '#services/FileStore' import { decodeSegment, type DecodedSegment } from '#lib/yenc/index' import type { RarInfo } from '#lib/rar/index' import { checkHeader, listingStart, parseBsdtarListing, parseSevenZipStart, sevenZipVolume, type SevenZipStart, } from '#lib/sevenzip/index' const run = promisify(execFile) /** The tail read for a listing: header plus packed header; more is not a header. */ const MAX_TAIL_BYTES = 16 * 1024 * 1024 /** * Lists 7z archives, also split ones (.7z.001, .002, …), from the first * segment of the first volume and the last segment(s) of the last one * (CONCEPT.md D19). Those bytes are put at their place in a sparse file the * size of the whole archive, which bsdtar (libarchive) lists: it only reads * the start and the header at the end. */ class SevenZipService { /** * The volumes of the archive `file` belongs to, in order, when all of them * up to the highest number seen are there; null otherwise. An unsplit (or * unnamed) archive is its own only volume. */ async volumes(file: any): Promise { const name = String(file.real_name ?? file.filename) const volume = sevenZipVolume(name) if (!volume) return [file] if (!file.release_id) return null const base = name.replace(/\.\d+$/, '').toLowerCase() const rows = await db .from('files') .where('release_id', file.release_id) .whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`]) const byNumber = new Map() for (const row of rows) { const n = sevenZipVolume(String(row.real_name ?? row.filename)) if (n && !byNumber.has(n)) byNumber.set(n, row) } const max = Math.max(0, ...byNumber.keys()) const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1)) return max > 0 && ordered.every(Boolean) ? ordered : null } /** * Lists the archive whose first volume starts with `firstData` (a decoded * first segment). Null when it isn't a 7z start, when volumes are missing, * or when the tail doesn't hold the header its start points to. */ async list(firstFile: any, firstData: Buffer): Promise { const start = parseSevenZipStart(firstData) if (!start) return null const volumes = await this.volumes(firstFile) if (!volumes) return null if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1) const tail = await this.readTail(volumes[volumes.length - 1], start) if (!tail) return null const dir = await mkdtemp(path.join(tmpdir(), '7z-')) try { const archive = path.join(dir, 'archive.7z') const handle = await open(archive, 'w') try { await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0) await handle.write(tail.data, 0, tail.data.length, tail.from) await handle.truncate(start.archiveSize) } finally { await handle.close() } return await this.bsdtar(archive, volumes.length > 1) } finally { await rm(dir, { recursive: true, force: true }) } } /** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */ private async bsdtar(archive: string, split: boolean): Promise { let stdout = '' try { stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout } catch (error: any) { if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split) logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`) return null } const entries = parseBsdtarListing(stdout) const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0] let encrypted = false if (sample) { try { await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' }) } catch (error: any) { // Without the data (it's all zeros here), anything else fails too: only this message counts. encrypted = /content is encrypted/i.test(String(error.stderr)) } } return this.info(entries, false, encrypted, split) } private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo { return { format: '7z', volume: split, firstVolume: true, solid: false, headersEncrypted, truncated: false, entries: entries.map((e) => ({ name: e.name, size: e.size, packedSize: 0, directory: e.directory, encrypted: encrypted && !e.directory, splitBefore: false, splitAfter: false, stored: false, dataOffset: 0, })), } } /** * The archive's tail from the last volume: the header (checked against its * CRC) and, for an encoded header, the packed stream before it. `from` is * its position in the whole archive. */ private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> { const ids = new Map(segmentsOf(last).map((s) => [s.number, s])) const numbers = [...ids.keys()] if (numbers.length === 0) return null const segments = new Map() const segment = async (n: number) => { if (!segments.has(n)) { const id = ids.get(n)?.id if (!id) throw new Error(`part ${n} of ${last.filename} is missing`) segments.set(n, decodeSegment(await this.body(id), { lenientSize: true })) } return segments.get(n)! } const lastNumber = numbers[numbers.length - 1] const lastSegment = await segment(lastNumber) const volumeSize = lastSegment.meta.begin.size const volumeStart = start.archiveSize - volumeSize const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1 const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null // Bytes [from, to) of the whole archive, from the last volume's parts. const read = async (from: number, to: number): Promise => { if (from < volumeStart) return null const a = from - volumeStart const b = to - volumeStart const pieces: Buffer[] = [] for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) { const s = await segment(n) const offset = (s.meta.part?.begin ?? 1) - 1 if (offset !== (n - 1) * partSize) return null pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset)))) } const data = Buffer.concat(pieces) return data.length === b - a ? data : null } const header = await read(start.headerStart, start.archiveSize) if (!header || !checkHeader(start, header)) return null const from = Math.min(listingStart(start, header), start.headerStart) if (start.archiveSize - from > MAX_TAIL_BYTES) return null const data = await read(from, start.archiveSize) return data ? { from, data } : null } private async body(messageId: string): Promise { const conn = await NntpService.acquire() let failed = false try { return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`) } catch (error: any) { failed = !/^4\d\d/.test(String(error.message)) throw error } finally { NntpService.release(conn, failed) } } } export default new SevenZipService()