Content inspection and archive walks on seeding nodes (relay)
Test and deploy / test (push) Successful in 1m49s
Test and deploy / deploy (push) Successful in 2m15s

Inspection is split into prepare (database), probe (lib/content/probe:
downloads, decoding and parsing, no database) and apply (database).
worker:collection runs all three; seed:node with SEED_INSPECTORS leases
jobs from the same queues through the seed API (BullMQ manual leases),
fetches articles through the seed API and posts the probes back.

A node now needs only SEED_URL and SEED_TOKEN: the settings the app
requires but a node never uses get placeholders for seed:node.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 11:17:02 +02:00
1 parent 51975fe603
commit 2dbca9b152
22 files changed
+1198 -540

No files matched your search

+1 -43
View File
@@ -1,12 +1,7 @@
import { execFile } from 'node:child_process'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import IngestService from '#services/IngestService'
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
import type { RarInfo } from '#lib/rar/index'
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
/** Stored per file and release; a listing of thousands of files is not useful. */
@@ -71,43 +66,6 @@ class ArchiveService {
}
}
const run = promisify(execFile)
/** Tools that print one archive entry to stdout, tried in order. */
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
]
/**
* The unpacked data of one entry of a complete (single-volume) archive:
* copied out when stored, else unpacked with bsdtar (libarchive, in the
* Docker image) or unrar. Null when that is not possible.
*/
export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
const stored = storedEntryData(archive, entry)
if (stored) return stored
if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
if (entry.size !== null && entry.size > maxBytes) return null
const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
try {
const file = path.join(dir, 'archive.rar')
await writeFile(file, archive)
for (const [tool, args] of EXTRACTORS) {
try {
const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
if (stdout.length > 0) return stdout
} catch (error: any) {
if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
}
}
return null
} finally {
await rm(dir, { recursive: true, force: true })
}
}
/**
* A release name from an archive listing: the top-level directory shared by
* all entries when it reads like a release name, otherwise the stem of the