Daan MeijerandClaude Opus 5.5
2dbca9b152
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
49 lines
1.8 KiB
TypeScript
49 lines
1.8 KiB
TypeScript
import { execFile } from 'node:child_process'
|
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import path from 'node:path'
|
|
import { promisify } from 'node:util'
|
|
import { storedEntryData, type RarEntry } from './parse.js'
|
|
|
|
const run = promisify(execFile)
|
|
|
|
/** Tools that print one archive entry to stdout, tried in order. */
|
|
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
|
|
['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
|
|
['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
|
|
]
|
|
|
|
/**
|
|
* The unpacked data of one entry of a complete (single-volume) archive:
|
|
* copied out when stored, else unpacked with bsdtar (libarchive, in the
|
|
* Docker image) or unrar. Null when that is not possible.
|
|
*/
|
|
export async function extractEntry(
|
|
archive: Buffer,
|
|
entry: RarEntry,
|
|
maxBytes = 64 * 1024 * 1024,
|
|
warn: (message: string) => void = () => {}
|
|
): Promise<Buffer | null> {
|
|
const stored = storedEntryData(archive, entry)
|
|
if (stored) return stored
|
|
if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
|
if (entry.size !== null && entry.size > maxBytes) return null
|
|
|
|
const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
|
|
try {
|
|
const file = path.join(dir, 'archive.rar')
|
|
await writeFile(file, archive)
|
|
for (const [tool, args] of EXTRACTORS) {
|
|
try {
|
|
const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
|
|
if (stdout.length > 0) return stdout
|
|
} catch (error: any) {
|
|
if (error.code !== 'ENOENT') warn(`${tool} could not extract "${entry.name}": ${error.message}`)
|
|
}
|
|
}
|
|
return null
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true })
|
|
}
|
|
}
|