import { createHash } from 'node:crypto' import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import IngestService from '#services/IngestService' import { type ReleaseIdentity } from '#lib/release/index' import { parseNzb, type ParsedNzb } from '#lib/nzb/index' import { parseSubject } from '#lib/subject/index' import { fileStem } from '#lib/release/index' import { looksRandom } from '#services/ArchiveService' export interface NzbImportResult { releaseId: number releaseName: string listed: number known: number added: number password: boolean } /** Filename of an NZB entry, from its subject (tolerating odd quoting). */ export function nzbFileName(subject: string): string { const parsed = parseSubject(subject) const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim() } export interface NzbImportOptions { /** * For NZBs inside an archive: the entry's name. It keeps several NZBs of * one archive apart and names their release when the NZB has no name. */ entryName?: string /** Move the posted file into the release the NZB describes (default true). */ moveContainer?: boolean /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */ name?: string /** Release source for new releases (default `nzb`). */ source?: ReleaseIdentity['source'] } /** * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26). * Files it lists that are already known (by first segment) keep their * release, which the NZB then joins; the others are added as files, so posts * spread over groups we do not index still become complete. */ class NzbImportService { async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise { const nzb = parseNzb(xml) if (!nzb || nzb.files.length === 0) return null const entries = nzb.files.map((f) => ({ file: f, name: nzbFileName(f.subject), firstId: f.segments[0].id, })) return db.transaction(async (trx) => { const known = await trx .from('files') .whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)]) .select('id', 'release_id', `message_ids`) const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id)) // Join the release the listed files are already in, else the posted NZB's own, else a new one. const counts = new Map() for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1) let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName) if (!releaseId) { const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}` const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex') const ids = await IngestService.upsertReleases(trx.knexClient, [ { release: { key, name: title, source: options.source ?? 'nzb' }, poster: nzbFile.poster, groups: nzbFile.groups ?? [], postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined, }, ]) releaseId = ids.get(key)! } const now = new Date() const missing = entries.filter((e) => !knownIds.has(e.firstId)) for (let i = 0; i < missing.length; i += 500) { await trx .knexClient('files') .insert( missing.slice(i, i + 500).map(({ file, name, firstId }) => ({ file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'), filename: name, poster: file.poster || nzbFile.poster, date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000, parts: file.segments.length, size: file.segments.reduce((sum, s) => sum + s.bytes, 0), message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))), groups: JSON.stringify(file.groups), release_id: releaseId, created_at: now, updated_at: now, })) ) .onConflict('file_key') .ignore() } // The poster's NZB says which files form the post: known files join this // release, also when name rules had split them over several. const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))] await trx .from('files') .whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id)) .update({ release_id: releaseId }) if (previous.length) { await trx.rawQuery( `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`, [releaseId, releaseId, releaseId] ) await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete() } // The NZB file joins the release it describes, unless it is part of a real release already. const own = nzbFile.release_id ? Number(nzbFile.release_id) : null if (own !== releaseId && options.moveContainer !== false) { const ownRelease = own ? await trx.from('releases').where('id', own).first() : null if (!ownRelease || ownRelease.file_count <= 1) { await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId }) if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete() } } const release = await trx.from('releases').where('id', releaseId).first() const update: Record = {} if (nzb.meta.password) update.password = nzb.meta.password if (!release.file_total) update.file_total = entries.length if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update) const result = { releaseId, releaseName: (update.name as string) ?? release.name, listed: entries.length, known: known.length, added: missing.length, password: Boolean(nzb.meta.password), } logger.info( `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` + `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.` ) return result }) } /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */ private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string { if (nzb.meta.name) return nzb.meta.name const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : '' if (entry && !looksRandom(entry)) return entry const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? '')) if (own && !looksRandom(own)) return own const largest = nzb.files .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) })) .sort((a, b) => b.bytes - a.bytes)[0] return fileStem(largest?.name ?? own) } } export default new NzbImportService()