usenet-indexer/app/services/SearchService.ts

import db from '@adonisjs/lucid/services/db'
import { buildNzb } from '#lib/nzb/index'
import SpotService from '#services/SpotService'
import { segmentsOf } from '#services/FileStore'
import { spotCategoryLabels } from '#lib/spotnet/index'

/**
 * `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
 * without asking); `missing`: the archive is encrypted and we don't have
 * it; `none`: no password needed as far as we know.
 */
export type PasswordState = 'known' | 'missing' | 'none'

export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
  if (hasPassword) return 'known'
  return encrypted ? 'missing' : 'none'
}

export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'

export interface SearchParams {
  q?: string
  /** Only releases whose name is readable (not an obfuscated token). */
  readable?: boolean
  /** Only releases that have all their files. */
  complete?: boolean
  /** Hide releases that need a password we don't have. */
  hideEncrypted?: boolean
  source?: string
  poster?: string
  sort?: SortOrder
  page?: number
  perPage?: number
  /** Newznab: exact category ids (already expanded from parents). */
  categories?: number[]
  /** Newznab: leave out releases known to be missing files. */
  notIncomplete?: boolean
  maxAgeDays?: number
  /** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
  season?: number
  episode?: number
  /** Overrides `page` (Newznab uses offsets). */
  offset?: number
  /** Leave out releases that are only an encrypted container (nzbe). */
  hideContainers?: boolean
  /** Include incomplete releases that consist of par2 files only (hidden by default). */
  includePar2Only?: boolean
  /** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
  countLimit?: number
}

const SORTS: Record<SortOrder, string> = {
  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
  largest: 'r.size DESC, r.id DESC',
  files: 'r.file_count DESC, r.id DESC',
  name: 'r.name ASC, r.id ASC',
}

/**
 * Shown in search: releases with files, and Spotnet listings whose NZB is
 * fetched on first download (CONCEPT.md D29). Matches the partial indexes.
 */
const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`

/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
  SELECT 1 FROM files f WHERE f.release_id = r.id
    AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`

/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
function pattern(word: string): string {
  const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
  return `%${escaped.replace(/[._\- ]+/g, '_')}%`
}

/**
 * As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
 * spaces as one space. `The.Block.AU` and `the block au` search the same.
 */
export function searchNormalize(text: string): string {
  return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
}

/** How long the web UI's totals are reused. */
const STATS_TTL_MS = 60 * 1000

/**
 * Counting stops here: a broad search needn't count every match. Each
 * counted row is a random heap read, so the web UI counts less (it only
 * shows "1,000+"); Newznab clients page by this total and keep the default.
 */
const COUNT_LIMIT = 10_000

/**
 * Searching and browsing releases for the web interface (CONCEPT.md D22).
 */
class SearchService {
  async search(params: SearchParams) {
    const where: string[] = [
      VISIBLE_SQL,
      // Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
    ]
    const bindings: unknown[] = []

    // Every word must occur in the release's search text: its name, archive
    // listing and file names (CONCEPT.md D33), one trigram index for all.
    const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
    for (const word of words) {
      where.push(`r.search_text LIKE ?`)
      bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
    }
    if (params.readable) where.push('is_readable_name(r.name)')
    if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
    if (params.hideContainers) where.push('r.container IS NULL')
    // par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
    if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
    // Kept current by triggers (migration …860).
    if (params.complete) where.push('r.complete')
    if (params.source) {
      where.push('r.source = ?')
      bindings.push(params.source)
    }
    if (params.poster) {
      where.push('r.poster ILIKE ?')
      bindings.push(pattern(params.poster))
    }
    if (params.categories?.length) {
      where.push('r.category = ANY(?)')
      bindings.push(params.categories)
    }
    if (params.notIncomplete) {
      where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
    }
    if (params.maxAgeDays) {
      where.push(`r.posted_at >= now() - make_interval(days => ?)`)
      bindings.push(params.maxAgeDays)
    }
    if (params.season !== undefined) {
      const s = params.season
      const e = params.episode
      where.push('r.name ~* ?')
      bindings.push(
        e !== undefined
          ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
          : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
      )
    }

    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
    const page = Math.max(params.page ?? 1, 1)
    const offset = params.offset ?? (page - 1) * perPage
    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
    const whereSql = where.join(' AND ')
    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)

    const [{ rows }, { rows: countRows }] = await Promise.all([
      db.rawQuery(
        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
                r.size, r.encrypted, is_readable_name(r.name) AS readable,
                (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
                (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
                   SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
                   WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
                jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
                r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
                (SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
         FROM releases r WHERE ${whereSql}
         ORDER BY ${order} LIMIT ? OFFSET ?`,
        [...bindings, perPage, offset]
      ),
      db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
    ])

    return {
      page,
      perPage,
      offset,
      total: Math.min(Number(countRows[0].total), countLimit),
      /** More than `total` match (the count stops at `countLimit`). */
      totalCapped: Number(countRows[0].total) > countLimit,
      releases: rows.map((r: any) => ({
        id: Number(r.id),
        name: r.name,
        source: r.source,
        poster: r.poster,
        groups: r.groups,
        postedAt: r.posted_at,
        fileTotal: r.file_total,
        fileCount: r.file_count,
        size: Number(r.size),
        encrypted: r.encrypted,
        readable: r.readable,
        listedFiles: Number(r.listed_files),
        listedPresent: Number(r.listed_present),
        archiveEntries: Number(r.archive_entries),
        complete: r.complete,
        category: r.category,
        container: r.container,
        par2Only: r.par2_only,
        listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
        hasPassword: r.has_password,
        passwordState: passwordState(r.encrypted, r.has_password),
        spotDescription: r.spot_description,
      })),
    }
  }

  async release(id: number) {
    const release = await db.from('releases').where('id', id).first()
    if (!release) return null

    const [files, listed, pending] = await Promise.all([
      db
        .from('files as f')
        .leftJoin('binaries as b', 'b.file_id', 'f.id')
        .where('f.release_id', id)
        .orderByRaw('coalesce(f.real_name, f.filename)')
        .select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
      db.rawQuery(
        `SELECT rf.name, rf.size,
                EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
                        WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
         FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
        [id]
      ),
      db.rawQuery(
        `SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
                coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
         FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
        [id]
      ),
    ])

    const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
    const spot = spotRow ? await SpotService.withDetails(spotRow) : null
    return {
      id: Number(release.id),
      name: release.name,
      source: release.source,
      spot: spot
        ? {
            messageId: spot.message_id,
            title: spot.title,
            tag: spot.tag,
            poster: spot.poster,
            postedAt: spot.posted_at,
            website: spot.website,
            description: spot.description,
            image: spot.image?.segments
              ? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
              : spot.image?.url
                ? { external: spot.image.url }
                : null,
            categories: spotCategoryLabels(spot.category, spot.subcats),
          }
        : null,
      password: release.password ?? null,
      passwordSource: release.password_source ?? null,
      passwordState: passwordState(release.encrypted, release.password !== null),
      listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
      poster: release.poster,
      groups: release.groups,
      postedAt: release.posted_at,
      fileTotal: release.file_total,
      fileCount: release.file_count,
      size: Number(release.size),
      encrypted: release.encrypted,
      container: release.container,
      contents: release.contents ?? [],
      files: files.map((f) => ({
        id: Number(f.id),
        name: f.real_name ?? f.filename,
        postedName: f.filename,
        realName: f.real_name,
        size: Number(f.exact_size ?? f.size),
        parts: f.parts,
        postedAt: f.date ? new Date(Number(f.date)) : null,
        poster: f.poster,
      })),
      par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
      pending: {
        binaries: Number(pending.rows[0].binaries),
        partsHave: Number(pending.rows[0].parts_have),
        partsTotal: Number(pending.rows[0].parts_total),
      },
    }
  }

  /** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
  async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
    const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
    if (!spot) return null
    const data = await SpotService.image(spot)
    if (!data) return null
    const type =
      data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
      : data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
      : data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
      : data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
      : data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
      : null
    return type ? { data, type } : null
  }

  async nzb(id: number): Promise<{ name: string; xml: string } | null> {
    let release = await db.from('releases').where('id', id).first()
    if (!release) return null
    if (release.source === 'spot' && Number(release.file_count) === 0) {
      // A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
      const imported = await SpotService.importListing(id)
      if (!imported) return null
      id = imported
      release = await db.from('releases').where('id', id).first()
    }
    const files = await db
      .from('files')
      .where('release_id', id)
      .orderByRaw('coalesce(real_name, filename)')
      .select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
    const xml = buildNzb(
      files.map((f) => ({
        name: f.real_name ?? f.filename,
        poster: f.poster,
        date: Number(f.date),
        groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
        segments: segmentsOf(f),
      })),
      { title: release.name, password: release.password ?? undefined }
    )
    return { name: release.name, xml }
  }

  /**
   * Totals for the web UI header. They count millions of rows (minutes with
   * the backfill running), so they are computed at most once per
   * STATS_TTL_MS and never twice at a time; callers in between share it.
   */
  async stats() {
    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
      this.statsRefresh ??= this.computeStats()
        .then((value) => {
          this.statsCache = { at: Date.now(), value }
        })
        // A failed background refresh keeps the previous totals; only a caller that waits sees the error.
        .catch((error) => {
          if (!this.statsCache) throw error
        })
        .finally(() => {
          this.statsRefresh = undefined
        })
      // Only the very first caller waits; later ones get the previous totals meanwhile.
      if (!this.statsCache) await this.statsRefresh
    }
    return this.statsCache!.value
  }

  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
  private statsRefresh?: Promise<void>

  /**
   * Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
   * from Postgres's statistics (visible releases = the entries of an index
   * that covers exactly them), ratios and sizes from a 0.5% sample of the
   * releases table. Within a few percent once autovacuum has analysed.
   */
  private async computeStats() {
    const { rows } = await db.rawQuery(`
      WITH sample AS (
        SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
        FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
      SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
             (SELECT reltuples FROM pg_class WHERE relname = 'files') AS files,
             (SELECT count(*) FROM sample) AS sampled,
             (SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
             -- Indexed data only: a spot listing's size is what the spot announces.
             (SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
             (SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
             (SELECT max(updated_at) FROM groups) AS last_indexed_at,
             (SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
                FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
    const r = rows[0]
    const releases = Math.max(0, Number(r.releases))
    const sampled = Number(r.sampled)
    // Sample counts scaled to all visible releases.
    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
    return {
      approximate: true,
      releases,
      readableReleases: scale(r.sampled_readable),
      files: Math.max(0, Number(r.files)),
      size: scale(r.sampled_size),
      encryptedReleases: scale(r.sampled_encrypted),
      lastIndexedAt: r.last_indexed_at,
      sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
    }
  }
}

export default new SearchService()