Content inspection and archive walks on seeding nodes (relay)
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51975fe603
commit
2dbca9b152
22 files changed
+1198
-540
No files matched your search
@@ -1,12 +1,7 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
|
||||
import type { RarInfo } from '#lib/rar/index'
|
||||
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
|
||||
|
||||
/** Stored per file and release; a listing of thousands of files is not useful. */
|
||||
@@ -71,43 +66,6 @@ class ArchiveService {
|
||||
}
|
||||
}
|
||||
|
||||
const run = promisify(execFile)
|
||||
|
||||
/** Tools that print one archive entry to stdout, tried in order. */
|
||||
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
|
||||
['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
|
||||
['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
|
||||
]
|
||||
|
||||
/**
|
||||
* The unpacked data of one entry of a complete (single-volume) archive:
|
||||
* copied out when stored, else unpacked with bsdtar (libarchive, in the
|
||||
* Docker image) or unrar. Null when that is not possible.
|
||||
*/
|
||||
export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
|
||||
const stored = storedEntryData(archive, entry)
|
||||
if (stored) return stored
|
||||
if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
||||
if (entry.size !== null && entry.size > maxBytes) return null
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
|
||||
try {
|
||||
const file = path.join(dir, 'archive.rar')
|
||||
await writeFile(file, archive)
|
||||
for (const [tool, args] of EXTRACTORS) {
|
||||
try {
|
||||
const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
|
||||
if (stdout.length > 0) return stdout
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
|
||||
}
|
||||
}
|
||||
return null
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A release name from an archive listing: the top-level directory shared by
|
||||
* all entries when it reads like a release name, otherwise the stem of the
|
||||
|
||||
Reference in new issue
Block a user