import { createHash } from 'node:crypto' import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import IngestService from '#services/IngestService' import { type ReleaseIdentity } from '#lib/release/index' import { parseNzb, type ParsedNzb } from '#lib/nzb/index' import { parseSubject } from '#lib/subject/index' import { fileStem } from '#lib/release/index' import { looksRandom } from '#services/ArchiveService' import QueueService from '#services/QueueService' import FileStore from '#services/FileStore' import { isArchiveStart } from '#lib/content/index' export interface NzbImportResult { releaseId: number releaseName: string listed: number known: number added: number password: boolean } /** Filename of an NZB entry, from its subject (tolerating odd quoting). */ export function nzbFileName(subject: string): string { const parsed = parseSubject(subject) const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim() return unquotedFileName(subject) } /** * A filename from a subject without quotes, as some spot NZBs have: * `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`, * `06 - Song.flac Artist - Album (2020) (01/35)`. Counters, "yEnc" and a * trailing size are dropped; then the name ends at the first extension * followed by a space. */ export function unquotedFileName(subject: string): string { const cleaned = subject .replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ') .replace(/\s\d+\s*$/, ' ') .replace(/(\s+-)+\s*$/, '') .replace(/^\s*-\s+/, '') .replace(/\s+/g, ' ') .trim() const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned) const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned return name.replace(/\s+-$/, '').trim() || subject.trim() } export interface NzbImportOptions { /** * For NZBs inside an archive: the entry's name. It keeps several NZBs of * one archive apart and names their release when the NZB has no name. */ entryName?: string /** Move the posted file into the release the NZB describes (default true). */ moveContainer?: boolean /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */ name?: string /** Release source for new releases (default `nzb`). */ source?: ReleaseIdentity['source'] /** * The NZB is authoritative for which files form the release and what it is * called (Spotnet spots): its files get the NZB's own release, named * `name`, also when name rules had put them in another release. Files * another trusted NZB claimed first stay where they are. */ trusted?: boolean } /** * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26). * Files it lists that are already known (by first segment) keep their * release, which the NZB then joins; the others are added as files, so posts * spread over groups we do not index still become complete. */ class NzbImportService { async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise { const nzb = parseNzb(xml) if (!nzb || nzb.files.length === 0) return null const entries = nzb.files.map((f) => ({ file: f, name: nzbFileName(f.subject), firstId: f.segments[0].id, })) const added: { id: number; filename: string }[] = [] const result = await db.transaction(async (trx) => { const known = await trx .from('files') .whereIn('first_message_id', entries.map((e) => e.firstId)) .select('id', 'release_id', 'first_message_id') const knownIds = new Set(known.map((f) => f.first_message_id)) // Join the release the listed files are already in, else the posted NZB's own, else a new one. const counts = new Map() for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1) let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName) const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}` const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex') // Releases of other trusted NZBs: their files are not taken over. let claimed = new Set() if (options.trusted) { // Only while that NZB still stands: a disposed spot's files go to its correction. const holders = await trx .from('releases as r') .whereIn('r.id', [...counts.keys()]) .where('r.source', 'spot') .whereNot('r.release_key', key) .whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy'])) .select('r.id') claimed = new Set(holders.map((r) => Number(r.id))) // Join another spot's release when it holds most of these files (a repost); else use our own. if (releaseId && !claimed.has(releaseId)) releaseId = undefined if (releaseId) claimed.delete(releaseId) } if (!releaseId) { const ids = await IngestService.upsertReleases(trx.knexClient, [ { release: { key, name: title, source: options.source ?? 'nzb' }, poster: nzbFile.poster, groups: nzbFile.groups ?? [], postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined, }, ]) releaseId = ids.get(key)! } const now = new Date() const missing = entries.filter((e) => !knownIds.has(e.firstId)) for (let i = 0; i < missing.length; i += 500) { added.push( ...(await FileStore.insert( missing.slice(i, i + 500).map(({ file, name, firstId }) => ({ file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'), filename: name, poster: file.poster || nzbFile.poster, date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000, parts: file.segments.length, size: file.segments.reduce((sum, s) => sum + s.bytes, 0), segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })), groups: JSON.stringify(file.groups), release_id: releaseId, created_at: now, updated_at: now, })), trx )) ) } // The poster's NZB says which files form the post: known files join this // release, also when name rules had split them over several. const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id))) const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))] await trx .from('files') .whereIn('id', moving.map((f) => f.id)) .update({ release_id: releaseId }) if (previous.length) { await trx.rawQuery( `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`, [releaseId, releaseId, releaseId] ) await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete() } // The NZB file joins the release it describes, unless it is part of a real release already. const own = nzbFile.release_id ? Number(nzbFile.release_id) : null if (own !== releaseId && options.moveContainer !== false) { const ownRelease = own ? await trx.from('releases').where('id', own).first() : null if (!ownRelease || ownRelease.file_count <= 1) { await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId }) if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete() } } const release = await trx.from('releases').where('id', releaseId).first() const update: Record = {} if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' }) if (!release.file_total) update.file_total = entries.length if (options.trusted) { // Only its own release: a repost joining another spot's release keeps that one's title. if (release.release_key === key && release.name !== title) update.name = title if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source } else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) { update.name = title } if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update) const result = { releaseId, releaseName: (update.name as string) ?? release.name, listed: entries.length, known: known.length, added: missing.length, password: Boolean(nzb.meta.password), } logger.info( `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` + `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.` ) return result }) // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB. const archives = added.filter((f) => isArchiveStart(f.filename)) if (archives.length) { await QueueService.collectionQueue.addBulk( archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } })) ) } return result } /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */ private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string { if (nzb.meta.name) return nzb.meta.name const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : '' if (entry && !looksRandom(entry)) return entry const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? '')) if (own && !looksRandom(own)) return own const largest = nzb.files .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) })) .sort((a, b) => b.bytes - a.bytes)[0] return fileStem(largest?.name ?? own) } } export default new NzbImportService()