usenet-indexer/app/services/ArchiveService.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import FileStore from '#services/FileStore'
import IngestService from '#services/IngestService'
import type { RarInfo } from '#lib/rar/index'
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'

/** Stored per file and release; a listing of thousands of files is not useful. */
const MAX_ENTRIES = 500

export interface ArchiveResult {
  info: RarInfo
  releaseId?: number
  releaseRenamed?: string
  grouped?: boolean
}

/**
 * Records what a RAR volume's headers say: the listing on the file, the
 * release it belongs to (from the shared directory or continued entry, when
 * the file has none yet), and for first volumes the contents and password
 * flag on the release. See CONCEPT.md (D19, D21).
 */
class ArchiveService {
  async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
    const { next: _next, ...listing } = info
    const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
    await FileStore.setDetails(Number(file.id), { archive: stored })

    let releaseId = file.release_id ?? binary?.release_id ?? null
    let grouped = false
    const identity = rarReleaseIdentity(info.entries)
    if (!releaseId && identity) {
      const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
        {
          release: identity,
          poster: binary?.poster ?? file.poster,
          weakIdentity: true,
          groups: binary?.groups ?? file.groups ?? [],
          postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
        },
      ])
      releaseId = ids.get(identity.key)!
      grouped = true
      if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
      await db.from('files').where('id', file.id).update({ release_id: releaseId })
    }
    if (!releaseId) return { info }

    const release = await db.from('releases').where('id', releaseId).first()
    const update: Record<string, unknown> = {}
    if (info.firstVolume !== false) {
      update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
      if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
    } else if (info.headersEncrypted) {
      update.encrypted = true
    }
    const better = releaseNameFromContents(info)
    // A spot's title is kept (CONCEPT.md D29).
    const renamed =
      release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
    if (renamed) update.name = renamed
    if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)

    if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
    return { info, releaseId, releaseRenamed: renamed, grouped }
  }
}

/**
 * A release name from an archive listing: the top-level directory shared by
 * all entries when it reads like a release name, otherwise the stem of the
 * largest entry with a readable name (e.g. an archive holding just
 * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
 */
export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
  // Hidden entries name nothing: one poster adds the same 22 GB ".1917-….mkv" decoy to every upload (LOG.md 2026-10-06 (2)).
  const listed = info.entries.filter((e) => !isHidden(e.name))
  const entries = listed.filter((e) => !e.directory)
  const tops = new Set(listed.map((e) => e.name.split(/[\\/]/)[0]))
  if (tops.size === 1 && listed.some((e) => /[\\/]/.test(e.name))) {
    const [top] = tops
    if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
  }
  const candidates = entries
    .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
    .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
    .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
    .sort((a, b) => b.size - a.size)
  return candidates[0]?.stem
}

/** An entry in a hidden directory or a hidden file itself (`.x/…`, `dir/.x`). */
export function isHidden(name: string): boolean {
  return name.split(/[\\/]/).some((part) => part.startsWith('.'))
}

/**
 * Releases named after a hidden archive entry before those were skipped:
 * renamed from their stored listing (no downloads). Used by
 * `release:rename --hidden`.
 */
export async function renameHiddenNamed(): Promise<number> {
  const releases = await db
    .from('releases')
    .where('name', 'like', '.%')
    .whereNot('source', 'spot')
    .whereNotNull('contents')
    .select('id', 'name', 'contents')
  let renamed = 0
  for (const release of releases) {
    const entries: any[] = Array.isArray(release.contents) ? release.contents : []
    const fromHidden = entries.some((e) => isHidden(e.name) && fileStem(String(e.name).split(/[\\/]/).pop() ?? '') === release.name)
    if (!fromHidden) continue
    const better = releaseNameFromContents({ entries })
    if (!better || better === release.name) continue
    await db.from('releases').where('id', release.id).update({ name: better })
    renamed++
  }
  return renamed
}

/**
 * Renames random-named releases from their stored archive listing (no
 * downloads). Used by `release:rename`.
 */
export async function renameFromStoredContents(): Promise<number> {
  const releases = await db
    .from('releases')
    .whereNotNull('contents')
    .whereRaw("jsonb_array_length(contents) > 0")
    .select('id', 'name', 'source', 'contents')
  let renamed = 0
  for (const release of releases) {
    if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
    const better = releaseNameFromContents({ entries: release.contents })
    if (!better || better === release.name) continue
    await db.from('releases').where('id', release.id).update({ name: better })
    renamed++
  }
  return renamed
}

/**
 * Obfuscated names: one long token of mixed letters and digits, no
 * separators, and the other generated shapes seen on Usenet (LOG.md
 * 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
 */
export function looksRandom(name: string): boolean {
  // An unknown short extension (e.g. ".bin") says nothing about the name.
  name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
  // VIDEO_TS, Sample, CD1: real words, but no name.
  if (isGenericName(name)) return true
  if (/^[0-9a-f]{16,}$/i.test(name)) return true
  // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
  if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
  // UUIDs
  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
  // 1650777167 (yEncBin), "83111763 n" (rclone uploads)
  if (/^\d{6,}( n)?$/.test(name)) return true
  // prvjr7hapd0i1ny14pz4, and rclone's long base32 names
  if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
  // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
  if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
  if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
  // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
  return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
}

export default new ArchiveService()