Daan MeijerandClaude Opus 5.5
35d0b2da65
5.4M waiting inspection jobs took 3.3 GB of Redis and most of its single thread. Producers now add file ids to inspection_queue; index-scheduler feeds up to INSPECTION_FEED_MAX (50k) into BullMQ. inspection:from-redis moves the existing backlog over once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
230 lines
10 KiB
TypeScript
230 lines
10 KiB
TypeScript
import { createHash } from 'node:crypto'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import IngestService from '#services/IngestService'
|
|
import { type ReleaseIdentity } from '#lib/release/index'
|
|
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
|
import { parseSubject } from '#lib/subject/index'
|
|
import { fileStem } from '#lib/release/index'
|
|
import { looksRandom } from '#services/ArchiveService'
|
|
import FileStore from '#services/FileStore'
|
|
import { isArchiveStart } from '#lib/content/index'
|
|
import InspectionQueue from '#services/InspectionQueue'
|
|
|
|
export interface NzbImportResult {
|
|
releaseId: number
|
|
releaseName: string
|
|
listed: number
|
|
known: number
|
|
added: number
|
|
password: boolean
|
|
}
|
|
|
|
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
|
|
export function nzbFileName(subject: string): string {
|
|
const parsed = parseSubject(subject)
|
|
const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
|
|
if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
|
return unquotedFileName(subject)
|
|
}
|
|
|
|
/**
|
|
* A filename from a subject without quotes, as some spot NZBs have:
|
|
* `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
|
|
* `06 - Song.flac Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
|
|
* trailing size are dropped; then the name ends at the first extension
|
|
* followed by a space.
|
|
*/
|
|
export function unquotedFileName(subject: string): string {
|
|
const cleaned = subject
|
|
.replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
|
|
.replace(/\s\d+\s*$/, ' ')
|
|
.replace(/(\s+-)+\s*$/, '')
|
|
.replace(/^\s*-\s+/, '')
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
|
|
const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
|
|
return name.replace(/\s+-$/, '').trim() || subject.trim()
|
|
}
|
|
|
|
export interface NzbImportOptions {
|
|
/**
|
|
* For NZBs inside an archive: the entry's name. It keeps several NZBs of
|
|
* one archive apart and names their release when the NZB has no name.
|
|
*/
|
|
entryName?: string
|
|
/** Move the posted file into the release the NZB describes (default true). */
|
|
moveContainer?: boolean
|
|
/** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
|
|
name?: string
|
|
/** Release source for new releases (default `nzb`). */
|
|
source?: ReleaseIdentity['source']
|
|
/**
|
|
* The NZB is authoritative for which files form the release and what it is
|
|
* called (Spotnet spots): its files get the NZB's own release, named
|
|
* `name`, also when name rules had put them in another release. Files
|
|
* another trusted NZB claimed first stay where they are.
|
|
*/
|
|
trusted?: boolean
|
|
}
|
|
|
|
/**
|
|
* Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
|
|
* Files it lists that are already known (by first segment) keep their
|
|
* release, which the NZB then joins; the others are added as files, so posts
|
|
* spread over groups we do not index still become complete.
|
|
*/
|
|
class NzbImportService {
|
|
async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
|
|
const nzb = parseNzb(xml)
|
|
if (!nzb || nzb.files.length === 0) return null
|
|
|
|
const entries = nzb.files.map((f) => ({
|
|
file: f,
|
|
name: nzbFileName(f.subject),
|
|
firstId: f.segments[0].id,
|
|
}))
|
|
|
|
const added: { id: number; filename: string }[] = []
|
|
const result = await db.transaction(async (trx) => {
|
|
const known = await trx
|
|
.from('files')
|
|
.whereIn('first_message_id', entries.map((e) => e.firstId))
|
|
.select('id', 'release_id', 'first_message_id')
|
|
const knownIds = new Set(known.map((f) => f.first_message_id))
|
|
|
|
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
|
|
const counts = new Map<number, number>()
|
|
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
|
|
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
|
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
|
|
const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
|
|
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
|
|
|
// Releases of other trusted NZBs: their files are not taken over.
|
|
let claimed = new Set<number>()
|
|
if (options.trusted) {
|
|
// Only while that NZB still stands: a disposed spot's files go to its correction.
|
|
const holders = await trx
|
|
.from('releases as r')
|
|
.whereIn('r.id', [...counts.keys()])
|
|
.where('r.source', 'spot')
|
|
.whereNot('r.release_key', key)
|
|
.whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
|
|
.select('r.id')
|
|
claimed = new Set(holders.map((r) => Number(r.id)))
|
|
// Join another spot's release when it holds most of these files (a repost); else use our own.
|
|
if (releaseId && !claimed.has(releaseId)) releaseId = undefined
|
|
if (releaseId) claimed.delete(releaseId)
|
|
}
|
|
if (!releaseId) {
|
|
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
|
{
|
|
release: { key, name: title, source: options.source ?? 'nzb' },
|
|
poster: nzbFile.poster,
|
|
groups: nzbFile.groups ?? [],
|
|
postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
|
|
},
|
|
])
|
|
releaseId = ids.get(key)!
|
|
}
|
|
|
|
const now = new Date()
|
|
const missing = entries.filter((e) => !knownIds.has(e.firstId))
|
|
for (let i = 0; i < missing.length; i += 500) {
|
|
added.push(
|
|
...(await FileStore.insert(
|
|
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
|
|
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
|
|
filename: name,
|
|
poster: file.poster || nzbFile.poster,
|
|
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
|
|
parts: file.segments.length,
|
|
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
|
|
segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
|
|
groups: JSON.stringify(file.groups),
|
|
release_id: releaseId,
|
|
created_at: now,
|
|
updated_at: now,
|
|
})),
|
|
trx
|
|
))
|
|
)
|
|
}
|
|
// The poster's NZB says which files form the post: known files join this
|
|
// release, also when name rules had split them over several.
|
|
const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
|
|
const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
|
|
await trx
|
|
.from('files')
|
|
.whereIn('id', moving.map((f) => f.id))
|
|
.update({ release_id: releaseId })
|
|
if (previous.length) {
|
|
await trx.rawQuery(
|
|
`UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
|
|
[releaseId, releaseId, releaseId]
|
|
)
|
|
await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
|
|
}
|
|
// The NZB file joins the release it describes, unless it is part of a real release already.
|
|
const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
|
|
if (own !== releaseId && options.moveContainer !== false) {
|
|
const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
|
|
if (!ownRelease || ownRelease.file_count <= 1) {
|
|
await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
|
|
if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
|
|
}
|
|
}
|
|
|
|
const release = await trx.from('releases').where('id', releaseId).first()
|
|
const update: Record<string, unknown> = {}
|
|
if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
|
|
if (!release.file_total) update.file_total = entries.length
|
|
if (options.trusted) {
|
|
// Only its own release: a repost joining another spot's release keeps that one's title.
|
|
if (release.release_key === key && release.name !== title) update.name = title
|
|
if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
|
|
} else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
|
|
update.name = title
|
|
}
|
|
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
|
|
|
|
const result = {
|
|
releaseId,
|
|
releaseName: (update.name as string) ?? release.name,
|
|
listed: entries.length,
|
|
known: known.length,
|
|
added: missing.length,
|
|
password: Boolean(nzb.meta.password),
|
|
}
|
|
logger.info(
|
|
`NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
|
|
`${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
|
|
)
|
|
return result
|
|
})
|
|
// Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
|
|
const archives = added.filter((f) => isArchiveStart(f.filename))
|
|
if (archives.length) {
|
|
await InspectionQueue.add(archives.map((f) => Number(f.id)))
|
|
}
|
|
return result
|
|
}
|
|
|
|
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
|
|
private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
|
|
if (nzb.meta.name) return nzb.meta.name
|
|
const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
|
|
if (entry && !looksRandom(entry)) return entry
|
|
const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
|
|
if (own && !looksRandom(own)) return own
|
|
const largest = nzb.files
|
|
.map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
|
|
.sort((a, b) => b.bytes - a.bytes)[0]
|
|
return fileStem(largest?.name ?? own)
|
|
}
|
|
}
|
|
|
|
export default new NzbImportService()
|