usenet-indexer/app/services/ContentService.ts

import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import NntpService from '#services/NntpService'
import Par2Service, { type Par2Result } from '#services/Par2Service'
import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
import NameService from '#services/NameService'
import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
import { sniffContentType, type ContentType } from '#lib/content/index'
import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
import { fileStem } from '#lib/release/index'
import { looksLikeFilename } from '#lib/subject/index'

const FIRST_CHUNK = 16 * 1024

/**
 * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
 * the file's first part. Undefined when that part is shorter than 16 KiB but
 * the file is not (the chunk would span two parts).
 */
export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
  return chunkMd5(decodeSegment(body).data, fileSize)
}

function chunkMd5(data: Buffer, fileSize: number): string | undefined {
  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
}

export interface ContentResult {
  par2?: Par2Result
  archive?: ArchiveResult
  named: number
  contentType: ContentType | null
  nzb?: NzbImportResult
  /** NZBs read from inside an archive. */
  archiveNzbs?: NzbImportResult[]
}

/**
 * Takes names and groupings from file contents (CONCEPT.md D21). For a
 * completed file with a random name, one article (the first) is enough:
 * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
 * - par2 files are recognised by content and name every file of their set,
 * - RAR headers give the release grouping, name and contents.
 */
class ContentService {
  /** Random or missing names, and named first RAR volumes (for contents and password flag). */
  isCandidate(name: string) {
    if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
    if (/\.nzb$/i.test(name)) return true
    return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
  }

  /** Downloads a whole (small) file whose first segment is already decoded. */
  private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
    const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
    const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
    if (parts.length === 1) return first

    let assembled: YencFile | undefined
    const conn = await NntpService.acquire()
    let failed = false
    try {
      for (const [, part] of parts) {
        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
        assembled ??= YencFile.fromSegment(segment)
        assembled.add(segment)
      }
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
    return assembled!.getBuffer()
  }

  /**
   * Downloads a small archive that lists NZBs and imports each of them
   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
   * uploads by other posters.
   */
  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
    const archive = await this.readWhole(first, file.message_ids)
    const info = archive && parseRarHeaders(archive)
    if (!archive || !info) return []
    const entries = info.entries.filter(isNzbEntry)
    const results: NzbImportResult[] = []
    for (const entry of entries) {
      const xml = await extractEntry(archive, entry)
      if (!xml) continue
      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
        entryName: entry.name,
        moveContainer: entries.length === 1,
      })
      if (imported) results.push(imported)
    }
    return results
  }

  async inspect(fileId: number): Promise<ContentResult | null> {
    const file = await db.from('files').where('id', fileId).first()
    const binary = await db.from('binaries').where('file_id', fileId).first()
    if (!file || !this.isCandidate(file.filename)) return null
    const first = file.message_ids?.['1']
    if (!first?.id) return null

    const conn = await NntpService.acquire()
    let failed = false
    let body: Buffer
    try {
      body = await NntpService.fetchBody(conn, first.id)
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
    let segment
    try {
      segment = decodeSegment(body)
    } catch (error) {
      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
      segment = decodeSegment(body, { lenientSize: true })
    }
    const size = segment.meta.begin.size
    const contentType = sniffContentType(segment.data)
    await db.from('files').where('id', fileId).update({ content_type: contentType })

    // Fingerprint, so par2 sets (now or later) can name this file exactly.
    if (binary) {
      await db
        .from('binaries')
        .where('id', binary.id)
        .update({
          size: db.raw('coalesce(size, ?)', [size]),
          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
        })
    }

    const result: ContentResult = { named: 0, contentType }
    if (contentType === 'nzbe' || contentType === 'rclone') {
      // An encrypted NZB, or rclone crypt (names and data encrypted): nothing
      // to read, and useless without the key.
      const releaseId = file.release_id ?? binary?.release_id
      if (releaseId) {
        await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
      }
      return result
    }
    if (contentType === 'nzb') {
      const xml = await this.readWhole(segment.data, file.message_ids)
      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
      return result
    }
    const set = await Par2Service.read(segment.data, file.message_ids)
    if (set && binary) {
      result.par2 = await Par2Service.attach(binary, set)
    } else {
      const info = parseRarHeaders(segment.data)
      if (info) {
        result.archive = await ArchiveService.record(file, binary, info)
        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
      }
    }
    if (binary) result.named = await NameService.resolve([binary.id])
    return result
  }
}

/** NZBs that can be read from a single, unencrypted archive. */
function hasNzbEntries(info: RarInfo): boolean {
  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
}

function isNzbEntry(entry: RarEntry): boolean {
  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
}

/** Small posted files (NZBs) that are read completely. */
const WHOLE_MAX_PARTS = 40
const WHOLE_MAX_BYTES = 25 * 1024 * 1024

export default new ContentService()