usenet-indexer/app/services/ArchiveService.ts

import { execFile } from 'node:child_process'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import IngestService from '#services/IngestService'
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
import { fileStem, rarReleaseIdentity } from '#lib/release/index'

/** Stored per file and release; a listing of thousands of files is not useful. */
const MAX_ENTRIES = 500

export interface ArchiveResult {
  info: RarInfo
  releaseId?: number
  releaseRenamed?: string
  grouped?: boolean
}

/**
 * Records what a RAR volume's headers say: the listing on the file, the
 * release it belongs to (from the shared directory or continued entry, when
 * the file has none yet), and for first volumes the contents and password
 * flag on the release. See CONCEPT.md (D19, D21).
 */
class ArchiveService {
  async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
    const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
    await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) })

    let releaseId = file.release_id ?? binary?.release_id ?? null
    let grouped = false
    const identity = rarReleaseIdentity(info.entries)
    if (!releaseId && identity) {
      const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
        {
          release: identity,
          poster: binary?.poster ?? file.poster,
          weakIdentity: true,
          groups: binary?.groups ?? file.groups ?? [],
          postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
        },
      ])
      releaseId = ids.get(identity.key)!
      grouped = true
      if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
      await db.from('files').where('id', file.id).update({ release_id: releaseId })
    }
    if (!releaseId) return { info }

    const release = await db.from('releases').where('id', releaseId).first()
    const update: Record<string, unknown> = {}
    if (info.firstVolume !== false) {
      update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
      if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
    } else if (info.headersEncrypted) {
      update.encrypted = true
    }
    const better = releaseNameFromContents(info)
    const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
    if (renamed) update.name = renamed
    if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)

    if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
    return { info, releaseId, releaseRenamed: renamed, grouped }
  }
}

const run = promisify(execFile)

/** Tools that print one archive entry to stdout, tried in order. */
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
  ['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
  ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
]

/**
 * The unpacked data of one entry of a complete (single-volume) archive:
 * copied out when stored, else unpacked with bsdtar (libarchive, in the
 * Docker image) or unrar. Null when that is not possible.
 */
export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
  const stored = storedEntryData(archive, entry)
  if (stored) return stored
  if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
  if (entry.size !== null && entry.size > maxBytes) return null

  const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
  try {
    const file = path.join(dir, 'archive.rar')
    await writeFile(file, archive)
    for (const [tool, args] of EXTRACTORS) {
      try {
        const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
        if (stdout.length > 0) return stdout
      } catch (error: any) {
        if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
      }
    }
    return null
  } finally {
    await rm(dir, { recursive: true, force: true })
  }
}

/**
 * A release name from an archive listing: the top-level directory shared by
 * all entries when it reads like a release name, otherwise the stem of the
 * largest entry with a readable name (e.g. an archive holding just
 * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
 */
export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
  const entries = info.entries.filter((e) => !e.directory)
  const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0]))
  if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) {
    const [top] = tops
    if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
  }
  const candidates = entries
    .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
    .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
    .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
    .sort((a, b) => b.size - a.size)
  return candidates[0]?.stem
}

/**
 * Renames random-named releases from their stored archive listing (no
 * downloads). Used by `release:rename`.
 */
export async function renameFromStoredContents(): Promise<number> {
  const releases = await db
    .from('releases')
    .whereNotNull('contents')
    .whereRaw("jsonb_array_length(contents) > 0")
    .select('id', 'name', 'source', 'contents')
  let renamed = 0
  for (const release of releases) {
    if (!looksRandom(release.name) && release.source !== 'post') continue
    const better = releaseNameFromContents({ entries: release.contents })
    if (!better || better === release.name) continue
    await db.from('releases').where('id', release.id).update({ name: better })
    renamed++
  }
  return renamed
}

/** Obfuscated names: one long token of mixed letters and digits, no separators. */
export function looksRandom(name: string): boolean {
  // An unknown short extension (e.g. ".bin") says nothing about the name.
  name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
  if (/^[0-9a-f]{16,}$/i.test(name)) return true
  if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
  // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
  return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
}

export default new ArchiveService()