implemented a lot of features

app/services/NzbImportService.ts

@@ -0,0 +1,168 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import IngestService from '#services/IngestService'
+import { type ReleaseIdentity } from '#lib/release/index'
+import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
+import { parseSubject } from '#lib/subject/index'
+import { fileStem } from '#lib/release/index'
+import { looksRandom } from '#services/ArchiveService'
+
+export interface NzbImportResult {
+  releaseId: number
+  releaseName: string
+  listed: number
+  known: number
+  added: number
+  password: boolean
+}
+
+/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
+export function nzbFileName(subject: string): string {
+  const parsed = parseSubject(subject)
+  const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
+  return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
+}
+
+export interface NzbImportOptions {
+  /**
+   * For NZBs inside an archive: the entry's name. It keeps several NZBs of
+   * one archive apart and names their release when the NZB has no name.
+   */
+  entryName?: string
+  /** Move the posted file into the release the NZB describes (default true). */
+  moveContainer?: boolean
+  /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
+  name?: string
+  /** Release source for new releases (default `nzb`). */
+  source?: ReleaseIdentity['source']
+}
+
+/**
+ * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
+ * Files it lists that are already known (by first segment) keep their
+ * release, which the NZB then joins; the others are added as files, so posts
+ * spread over groups we do not index still become complete.
+ */
+class NzbImportService {
+  async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
+    const nzb = parseNzb(xml)
+    if (!nzb || nzb.files.length === 0) return null
+
+    const entries = nzb.files.map((f) => ({
+      file: f,
+      name: nzbFileName(f.subject),
+      firstId: f.segments[0].id,
+    }))
+
+    return db.transaction(async (trx) => {
+      const known = await trx
+        .from('files')
+        .whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
+        .select('id', 'release_id', `message_ids`)
+      const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
+
+      // Join the release the listed files are already in, else the posted NZB's own, else a new one.
+      const counts = new Map<number, number>()
+      for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
+      let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
+      const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
+      if (!releaseId) {
+        const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
+        const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
+        const ids = await IngestService.upsertReleases(trx.knexClient, [
+          {
+            release: { key, name: title, source: options.source ?? 'nzb' },
+            poster: nzbFile.poster,
+            groups: nzbFile.groups ?? [],
+            postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
+          },
+        ])
+        releaseId = ids.get(key)!
+      }
+
+      const now = new Date()
+      const missing = entries.filter((e) => !knownIds.has(e.firstId))
+      for (let i = 0; i < missing.length; i += 500) {
+        await trx
+          .knexClient('files')
+          .insert(
+            missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
+              file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
+              filename: name,
+              poster: file.poster || nzbFile.poster,
+              date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
+              parts: file.segments.length,
+              size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
+              message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
+              groups: JSON.stringify(file.groups),
+              release_id: releaseId,
+              created_at: now,
+              updated_at: now,
+            }))
+          )
+          .onConflict('file_key')
+          .ignore()
+      }
+      // The poster's NZB says which files form the post: known files join this
+      // release, also when name rules had split them over several.
+      const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
+      await trx
+        .from('files')
+        .whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
+        .update({ release_id: releaseId })
+      if (previous.length) {
+        await trx.rawQuery(
+          `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
+          [releaseId, releaseId, releaseId]
+        )
+        await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
+      }
+      // The NZB file joins the release it describes, unless it is part of a real release already.
+      const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
+      if (own !== releaseId && options.moveContainer !== false) {
+        const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
+        if (!ownRelease || ownRelease.file_count <= 1) {
+          await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
+          if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
+        }
+      }
+
+      const release = await trx.from('releases').where('id', releaseId).first()
+      const update: Record<string, unknown> = {}
+      if (nzb.meta.password) update.password = nzb.meta.password
+      if (!release.file_total) update.file_total = entries.length
+      if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
+      if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
+
+      const result = {
+        releaseId,
+        releaseName: (update.name as string) ?? release.name,
+        listed: entries.length,
+        known: known.length,
+        added: missing.length,
+        password: Boolean(nzb.meta.password),
+      }
+      logger.info(
+        `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
+          `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
+      )
+      return result
+    })
+  }
+
+  /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
+  private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
+    if (nzb.meta.name) return nzb.meta.name
+    const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
+    if (entry && !looksRandom(entry)) return entry
+    const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
+    if (own && !looksRandom(own)) return own
+    const largest = nzb.files
+      .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
+      .sort((a, b) => b.bytes - a.bytes)[0]
+    return fileStem(largest?.name ?? own)
+  }
+}
+
+export default new NzbImportService()