usenet-indexer/app/services/NzbImportService.ts

import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import IngestService from '#services/IngestService'
import { type ReleaseIdentity } from '#lib/release/index'
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
import { parseSubject } from '#lib/subject/index'
import { fileStem } from '#lib/release/index'
import { looksRandom } from '#services/ArchiveService'
import QueueService from '#services/QueueService'
import FileStore from '#services/FileStore'
import { isArchiveStart } from '#lib/content/index'

export interface NzbImportResult {
  releaseId: number
  releaseName: string
  listed: number
  known: number
  added: number
  password: boolean
}

/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
export function nzbFileName(subject: string): string {
  const parsed = parseSubject(subject)
  const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
  if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
  return unquotedFileName(subject)
}

/**
 * A filename from a subject without quotes, as some spot NZBs have:
 * `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
 * `06 - Song.flac  Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
 * trailing size are dropped; then the name ends at the first extension
 * followed by a space.
 */
export function unquotedFileName(subject: string): string {
  const cleaned = subject
    .replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
    .replace(/\s\d+\s*$/, ' ')
    .replace(/(\s+-)+\s*$/, '')
    .replace(/^\s*-\s+/, '')
    .replace(/\s+/g, ' ')
    .trim()
  const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
  const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
  return name.replace(/\s+-$/, '').trim() || subject.trim()
}

export interface NzbImportOptions {
  /**
   * For NZBs inside an archive: the entry's name. It keeps several NZBs of
   * one archive apart and names their release when the NZB has no name.
   */
  entryName?: string
  /** Move the posted file into the release the NZB describes (default true). */
  moveContainer?: boolean
  /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
  name?: string
  /** Release source for new releases (default `nzb`). */
  source?: ReleaseIdentity['source']
  /**
   * The NZB is authoritative for which files form the release and what it is
   * called (Spotnet spots): its files get the NZB's own release, named
   * `name`, also when name rules had put them in another release. Files
   * another trusted NZB claimed first stay where they are.
   */
  trusted?: boolean
}

/**
 * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
 * Files it lists that are already known (by first segment) keep their
 * release, which the NZB then joins; the others are added as files, so posts
 * spread over groups we do not index still become complete.
 */
class NzbImportService {
  async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
    const nzb = parseNzb(xml)
    if (!nzb || nzb.files.length === 0) return null

    const entries = nzb.files.map((f) => ({
      file: f,
      name: nzbFileName(f.subject),
      firstId: f.segments[0].id,
    }))

    const added: { id: number; filename: string }[] = []
    const result = await db.transaction(async (trx) => {
      const known = await trx
        .from('files')
        .whereIn('first_message_id', entries.map((e) => e.firstId))
        .select('id', 'release_id', 'first_message_id')
      const knownIds = new Set(known.map((f) => f.first_message_id))

      // Join the release the listed files are already in, else the posted NZB's own, else a new one.
      const counts = new Map<number, number>()
      for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
      let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
      const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
      const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
      const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')

      // Releases of other trusted NZBs: their files are not taken over.
      let claimed = new Set<number>()
      if (options.trusted) {
        // Only while that NZB still stands: a disposed spot's files go to its correction.
        const holders = await trx
          .from('releases as r')
          .whereIn('r.id', [...counts.keys()])
          .where('r.source', 'spot')
          .whereNot('r.release_key', key)
          .whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
          .select('r.id')
        claimed = new Set(holders.map((r) => Number(r.id)))
        // Join another spot's release when it holds most of these files (a repost); else use our own.
        if (releaseId && !claimed.has(releaseId)) releaseId = undefined
        if (releaseId) claimed.delete(releaseId)
      }
      if (!releaseId) {
        const ids = await IngestService.upsertReleases(trx.knexClient, [
          {
            release: { key, name: title, source: options.source ?? 'nzb' },
            poster: nzbFile.poster,
            groups: nzbFile.groups ?? [],
            postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
          },
        ])
        releaseId = ids.get(key)!
      }

      const now = new Date()
      const missing = entries.filter((e) => !knownIds.has(e.firstId))
      for (let i = 0; i < missing.length; i += 500) {
        added.push(
          ...(await FileStore.insert(
            missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
              file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
              filename: name,
              poster: file.poster || nzbFile.poster,
              date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
              parts: file.segments.length,
              size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
              segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
              groups: JSON.stringify(file.groups),
              release_id: releaseId,
              created_at: now,
              updated_at: now,
            })),
            trx
          ))
        )
      }
      // The poster's NZB says which files form the post: known files join this
      // release, also when name rules had split them over several.
      const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
      const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
      await trx
        .from('files')
        .whereIn('id', moving.map((f) => f.id))
        .update({ release_id: releaseId })
      if (previous.length) {
        await trx.rawQuery(
          `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
          [releaseId, releaseId, releaseId]
        )
        await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
      }
      // The NZB file joins the release it describes, unless it is part of a real release already.
      const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
      if (own !== releaseId && options.moveContainer !== false) {
        const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
        if (!ownRelease || ownRelease.file_count <= 1) {
          await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
          if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
        }
      }

      const release = await trx.from('releases').where('id', releaseId).first()
      const update: Record<string, unknown> = {}
      if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
      if (!release.file_total) update.file_total = entries.length
      if (options.trusted) {
        // Only its own release: a repost joining another spot's release keeps that one's title.
        if (release.release_key === key && release.name !== title) update.name = title
        if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
      } else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
        update.name = title
      }
      if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)

      const result = {
        releaseId,
        releaseName: (update.name as string) ?? release.name,
        listed: entries.length,
        known: known.length,
        added: missing.length,
        password: Boolean(nzb.meta.password),
      }
      logger.info(
        `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
          `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
      )
      return result
    })
    // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
    const archives = added.filter((f) => isArchiveStart(f.filename))
    if (archives.length) {
      await QueueService.collectionQueue.addBulk(
        archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
      )
    }
    return result
  }

  /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
  private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
    if (nzb.meta.name) return nzb.meta.name
    const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
    if (entry && !looksRandom(entry)) return entry
    const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
    if (own && !looksRandom(own)) return own
    const largest = nzb.files
      .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
      .sort((a, b) => b.bytes - a.bytes)[0]
    return fileStem(largest?.name ?? own)
  }
}

export default new NzbImportService()