169 lines
7.6 KiB
TypeScript
169 lines
7.6 KiB
TypeScript
import { createHash } from 'node:crypto'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import IngestService from '#services/IngestService'
|
|
import { type ReleaseIdentity } from '#lib/release/index'
|
|
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
|
import { parseSubject } from '#lib/subject/index'
|
|
import { fileStem } from '#lib/release/index'
|
|
import { looksRandom } from '#services/ArchiveService'
|
|
|
|
export interface NzbImportResult {
|
|
releaseId: number
|
|
releaseName: string
|
|
listed: number
|
|
known: number
|
|
added: number
|
|
password: boolean
|
|
}
|
|
|
|
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
|
|
export function nzbFileName(subject: string): string {
|
|
const parsed = parseSubject(subject)
|
|
const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
|
|
return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
|
}
|
|
|
|
export interface NzbImportOptions {
|
|
/**
|
|
* For NZBs inside an archive: the entry's name. It keeps several NZBs of
|
|
* one archive apart and names their release when the NZB has no name.
|
|
*/
|
|
entryName?: string
|
|
/** Move the posted file into the release the NZB describes (default true). */
|
|
moveContainer?: boolean
|
|
/** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
|
|
name?: string
|
|
/** Release source for new releases (default `nzb`). */
|
|
source?: ReleaseIdentity['source']
|
|
}
|
|
|
|
/**
|
|
* Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
|
|
* Files it lists that are already known (by first segment) keep their
|
|
* release, which the NZB then joins; the others are added as files, so posts
|
|
* spread over groups we do not index still become complete.
|
|
*/
|
|
class NzbImportService {
|
|
async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
|
|
const nzb = parseNzb(xml)
|
|
if (!nzb || nzb.files.length === 0) return null
|
|
|
|
const entries = nzb.files.map((f) => ({
|
|
file: f,
|
|
name: nzbFileName(f.subject),
|
|
firstId: f.segments[0].id,
|
|
}))
|
|
|
|
return db.transaction(async (trx) => {
|
|
const known = await trx
|
|
.from('files')
|
|
.whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
|
|
.select('id', 'release_id', `message_ids`)
|
|
const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
|
|
|
|
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
|
|
const counts = new Map<number, number>()
|
|
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
|
|
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
|
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
|
|
if (!releaseId) {
|
|
const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
|
|
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
|
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
|
{
|
|
release: { key, name: title, source: options.source ?? 'nzb' },
|
|
poster: nzbFile.poster,
|
|
groups: nzbFile.groups ?? [],
|
|
postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
|
|
},
|
|
])
|
|
releaseId = ids.get(key)!
|
|
}
|
|
|
|
const now = new Date()
|
|
const missing = entries.filter((e) => !knownIds.has(e.firstId))
|
|
for (let i = 0; i < missing.length; i += 500) {
|
|
await trx
|
|
.knexClient('files')
|
|
.insert(
|
|
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
|
|
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
|
|
filename: name,
|
|
poster: file.poster || nzbFile.poster,
|
|
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
|
|
parts: file.segments.length,
|
|
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
|
|
message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
|
|
groups: JSON.stringify(file.groups),
|
|
release_id: releaseId,
|
|
created_at: now,
|
|
updated_at: now,
|
|
}))
|
|
)
|
|
.onConflict('file_key')
|
|
.ignore()
|
|
}
|
|
// The poster's NZB says which files form the post: known files join this
|
|
// release, also when name rules had split them over several.
|
|
const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
|
|
await trx
|
|
.from('files')
|
|
.whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
|
|
.update({ release_id: releaseId })
|
|
if (previous.length) {
|
|
await trx.rawQuery(
|
|
`UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
|
|
[releaseId, releaseId, releaseId]
|
|
)
|
|
await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
|
|
}
|
|
// The NZB file joins the release it describes, unless it is part of a real release already.
|
|
const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
|
|
if (own !== releaseId && options.moveContainer !== false) {
|
|
const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
|
|
if (!ownRelease || ownRelease.file_count <= 1) {
|
|
await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
|
|
if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
|
|
}
|
|
}
|
|
|
|
const release = await trx.from('releases').where('id', releaseId).first()
|
|
const update: Record<string, unknown> = {}
|
|
if (nzb.meta.password) update.password = nzb.meta.password
|
|
if (!release.file_total) update.file_total = entries.length
|
|
if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
|
|
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
|
|
|
|
const result = {
|
|
releaseId,
|
|
releaseName: (update.name as string) ?? release.name,
|
|
listed: entries.length,
|
|
known: known.length,
|
|
added: missing.length,
|
|
password: Boolean(nzb.meta.password),
|
|
}
|
|
logger.info(
|
|
`NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
|
|
`${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
|
|
)
|
|
return result
|
|
})
|
|
}
|
|
|
|
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
|
|
private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
|
|
if (nzb.meta.name) return nzb.meta.name
|
|
const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
|
|
if (entry && !looksRandom(entry)) return entry
|
|
const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
|
|
if (own && !looksRandom(own)) return own
|
|
const largest = nzb.files
|
|
.map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
|
|
.sort((a, b) => b.bytes - a.bytes)[0]
|
|
return fileStem(largest?.name ?? own)
|
|
}
|
|
}
|
|
|
|
export default new NzbImportService()
|