Content inspection and archive walks on seeding nodes (relay)
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
51975fe603
commit
2dbca9b152
22 files changed
+1198
-540
No files matched your search
@@ -1,35 +1,10 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { mkdtemp, open, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
|
||||
import type { RarInfo } from '#lib/rar/index'
|
||||
import {
|
||||
checkHeader,
|
||||
encodedHeaderEncrypted,
|
||||
listingStart,
|
||||
parseBsdtarListing,
|
||||
parseSevenZipStart,
|
||||
sevenZipVolume,
|
||||
type SevenZipStart,
|
||||
} from '#lib/sevenzip/index'
|
||||
|
||||
const run = promisify(execFile)
|
||||
|
||||
/** The tail read for a listing: header plus packed header; more is not a header. */
|
||||
const MAX_TAIL_BYTES = 16 * 1024 * 1024
|
||||
import { sevenZipVolume } from '#lib/sevenzip/index'
|
||||
|
||||
/**
|
||||
* Lists 7z archives, also split ones (.7z.001, .002, …), from the first
|
||||
* segment of the first volume and the last segment(s) of the last one
|
||||
* (CONCEPT.md D19). Those bytes are put at their place in a sparse file the
|
||||
* size of the whole archive, which bsdtar (libarchive) lists: it only reads
|
||||
* the start and the header at the end.
|
||||
* The database side of 7z listings (CONCEPT.md D19): which volumes make up
|
||||
* an archive. Listing itself (first segment, tail, bsdtar) is
|
||||
* lib/sevenzip/list.ts, run by inspection here or on a node.
|
||||
*/
|
||||
class SevenZipService {
|
||||
/**
|
||||
@@ -56,152 +31,6 @@ class SevenZipService {
|
||||
const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
|
||||
return max > 0 && ordered.every(Boolean) ? ordered : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists the archive whose first volume starts with `firstData` (a decoded
|
||||
* first segment). Null when it isn't a 7z start, when volumes are missing,
|
||||
* or when the tail doesn't hold the header its start points to.
|
||||
*/
|
||||
async list(firstFile: any, firstData: Buffer): Promise<RarInfo | null> {
|
||||
const start = parseSevenZipStart(firstData)
|
||||
if (!start) return null
|
||||
const volumes = await this.volumes(firstFile)
|
||||
if (!volumes) return null
|
||||
if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1)
|
||||
|
||||
const tail = await this.readTail(volumes[volumes.length - 1], start)
|
||||
if (!tail) return null
|
||||
// Encrypted headers (-mhe): known from the header alone, no listing possible (most 7z uploads).
|
||||
if ('headersEncrypted' in tail) return this.info([], true, true, volumes.length > 1)
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
|
||||
try {
|
||||
const archive = path.join(dir, 'archive.7z')
|
||||
const handle = await open(archive, 'w')
|
||||
try {
|
||||
await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
|
||||
await handle.write(tail.data, 0, tail.data.length, tail.from)
|
||||
await handle.truncate(start.archiveSize)
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return await this.bsdtar(archive, volumes.length > 1)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
|
||||
private async bsdtar(archive: string, split: boolean): Promise<RarInfo | null> {
|
||||
let stdout = ''
|
||||
try {
|
||||
stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
|
||||
} catch (error: any) {
|
||||
if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split)
|
||||
logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
|
||||
return null
|
||||
}
|
||||
const entries = parseBsdtarListing(stdout)
|
||||
const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
|
||||
let encrypted = false
|
||||
if (sample) {
|
||||
try {
|
||||
await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
|
||||
} catch (error: any) {
|
||||
// Without the data (it's all zeros here), anything else fails too: only this message counts.
|
||||
encrypted = /content is encrypted/i.test(String(error.stderr))
|
||||
}
|
||||
}
|
||||
return this.info(entries, false, encrypted, split)
|
||||
}
|
||||
|
||||
private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo {
|
||||
return {
|
||||
format: '7z',
|
||||
volume: split,
|
||||
firstVolume: true,
|
||||
solid: false,
|
||||
headersEncrypted,
|
||||
truncated: false,
|
||||
next: 0,
|
||||
entries: entries.map((e) => ({
|
||||
name: e.name,
|
||||
size: e.size,
|
||||
packedSize: 0,
|
||||
directory: e.directory,
|
||||
encrypted: encrypted && !e.directory,
|
||||
splitBefore: false,
|
||||
splitAfter: false,
|
||||
stored: false,
|
||||
dataOffset: 0,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The archive's tail from the last volume: the header (checked against its
|
||||
* CRC) and, for an encoded header, the packed stream before it. `from` is
|
||||
* its position in the whole archive.
|
||||
*/
|
||||
private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | { headersEncrypted: true } | null> {
|
||||
const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
|
||||
const numbers = [...ids.keys()]
|
||||
if (numbers.length === 0) return null
|
||||
const segments = new Map<number, DecodedSegment>()
|
||||
const segment = async (n: number) => {
|
||||
if (!segments.has(n)) {
|
||||
const id = ids.get(n)?.id
|
||||
if (!id) throw new Error(`part ${n} of ${last.filename} is missing`)
|
||||
segments.set(n, decodeSegment(await this.body(id), { lenientSize: true }))
|
||||
}
|
||||
return segments.get(n)!
|
||||
}
|
||||
|
||||
const lastNumber = numbers[numbers.length - 1]
|
||||
const lastSegment = await segment(lastNumber)
|
||||
const volumeSize = lastSegment.meta.begin.size
|
||||
const volumeStart = start.archiveSize - volumeSize
|
||||
const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
|
||||
const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
|
||||
if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
|
||||
|
||||
// Bytes [from, to) of the whole archive, from the last volume's parts.
|
||||
const read = async (from: number, to: number): Promise<Buffer | null> => {
|
||||
if (from < volumeStart) return null
|
||||
const a = from - volumeStart
|
||||
const b = to - volumeStart
|
||||
const pieces: Buffer[] = []
|
||||
for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
|
||||
const s = await segment(n)
|
||||
const offset = (s.meta.part?.begin ?? 1) - 1
|
||||
if (offset !== (n - 1) * partSize) return null
|
||||
pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
|
||||
}
|
||||
const data = Buffer.concat(pieces)
|
||||
return data.length === b - a ? data : null
|
||||
}
|
||||
|
||||
const header = await read(start.headerStart, start.archiveSize)
|
||||
if (!header || !checkHeader(start, header)) return null
|
||||
if (encodedHeaderEncrypted(header)) return { headersEncrypted: true }
|
||||
const from = Math.min(listingStart(start, header), start.headerStart)
|
||||
if (start.archiveSize - from > MAX_TAIL_BYTES) return null
|
||||
const data = await read(from, start.archiveSize)
|
||||
return data ? { from, data } : null
|
||||
}
|
||||
|
||||
private async body(messageId: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
|
||||
} catch (error: any) {
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SevenZipService()
|
||||
Reference in new issue
Block a user