import { createHash } from 'node:crypto' import db from '@adonisjs/lucid/services/db' import NntpService from '#services/NntpService' import Par2Service, { type Par2Result } from '#services/Par2Service' import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService' import NameService from '#services/NameService' import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index' import NzbImportService, { type NzbImportResult } from '#services/NzbImportService' import { sniffContentType, type ContentType } from '#lib/content/index' import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index' import { fileStem } from '#lib/release/index' import { looksLikeFilename } from '#lib/subject/index' const FIRST_CHUNK = 16 * 1024 /** * MD5 of the first 16 KiB of a file, as par2 records it, from the body of * the file's first part. Undefined when that part is shorter than 16 KiB but * the file is not (the chunk would span two parts). */ export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined { return chunkMd5(decodeSegment(body).data, fileSize) } function chunkMd5(data: Buffer, fileSize: number): string | undefined { if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex') } export interface ContentResult { par2?: Par2Result archive?: ArchiveResult named: number contentType: ContentType | null nzb?: NzbImportResult /** NZBs read from inside an archive. */ archiveNzbs?: NzbImportResult[] } /** * Takes names and groupings from file contents (CONCEPT.md D21). For a * completed file with a random name, one article (the first) is enough: * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry, * - par2 files are recognised by content and name every file of their set, * - RAR headers give the release grouping, name and contents. */ class ContentService { /** Random or missing names, and named first RAR volumes (for contents and password flag). */ isCandidate(name: string) { if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true if (/\.nzb$/i.test(name)) return true return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name)) } /** Downloads a whole (small) file whose first segment is already decoded. */ private async readWhole(first: Buffer, messageIds: Record): Promise { const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b)) const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0) if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null if (parts.length === 1) return first let assembled: YencFile | undefined const conn = await NntpService.acquire() let failed = false try { for (const [, part] of parts) { const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true }) assembled ??= YencFile.fromSegment(segment) assembled.add(segment) } } catch (error) { failed = true throw error } finally { NntpService.release(conn, failed) } return assembled!.getBuffer() } /** * Downloads a small archive that lists NZBs and imports each of them * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated * uploads by other posters. */ private async importArchiveNzbs(file: any, first: Buffer): Promise { const archive = await this.readWhole(first, file.message_ids) const info = archive && parseRarHeaders(archive) if (!archive || !info) return [] const entries = info.entries.filter(isNzbEntry) const results: NzbImportResult[] = [] for (const entry of entries) { const xml = await extractEntry(archive, entry) if (!xml) continue const imported = await NzbImportService.import(file, xml.toString('utf8'), { entryName: entry.name, moveContainer: entries.length === 1, }) if (imported) results.push(imported) } return results } async inspect(fileId: number): Promise { const file = await db.from('files').where('id', fileId).first() const binary = await db.from('binaries').where('file_id', fileId).first() if (!file || !this.isCandidate(file.filename)) return null const first = file.message_ids?.['1'] if (!first?.id) return null const conn = await NntpService.acquire() let failed = false let body: Buffer try { body = await NntpService.fetchBody(conn, first.id) } catch (error) { failed = true throw error } finally { NntpService.release(conn, failed) } // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match. let segment try { segment = decodeSegment(body) } catch (error) { if (!(error instanceof YencError) || error instanceof YencCrcError) throw error segment = decodeSegment(body, { lenientSize: true }) } const size = segment.meta.begin.size const contentType = sniffContentType(segment.data) await db.from('files').where('id', fileId).update({ content_type: contentType }) // Fingerprint, so par2 sets (now or later) can name this file exactly. if (binary) { await db .from('binaries') .where('id', binary.id) .update({ size: db.raw('coalesce(size, ?)', [size]), md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]), }) } const result: ContentResult = { named: 0, contentType } if (contentType === 'nzbe' || contentType === 'rclone') { // An encrypted NZB, or rclone crypt (names and data encrypted): nothing // to read, and useless without the key. const releaseId = file.release_id ?? binary?.release_id if (releaseId) { await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true }) } return result } if (contentType === 'nzb') { const xml = await this.readWhole(segment.data, file.message_ids) if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined return result } const set = await Par2Service.read(segment.data, file.message_ids) if (set && binary) { result.par2 = await Par2Service.attach(binary, set) } else { const info = parseRarHeaders(segment.data) if (info) { result.archive = await ArchiveService.record(file, binary, info) if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data) } } if (binary) result.named = await NameService.resolve([binary.id]) return result } } /** NZBs that can be read from a single, unencrypted archive. */ function hasNzbEntries(info: RarInfo): boolean { return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry) } function isNzbEntry(entry: RarEntry): boolean { return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory } /** Small posted files (NZBs) that are read completely. */ const WHOLE_MAX_PARTS = 40 const WHOLE_MAX_BYTES = 25 * 1024 * 1024 export default new ContentService()