usenet-indexer/app/services/search_service.ts

import db from '@adonisjs/lucid/services/db'
import NzbService from '#services/nzb_service'

export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'

/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
export interface SearchParams {
  q?: string
  /** Only releases with a file whose name is readable (not an obfuscated token). */
  readable?: boolean
  /** Only releases known to have all their files. */
  complete?: boolean
  /** Leave out releases known to miss files (Newznab). */
  notIncomplete?: boolean
  /** How the release was recognised: stem, subject or post. */
  source?: string
  poster?: string
  /** Exact category ids (already expanded from parents). */
  categories?: number[]
  maxAgeDays?: number
  /** TV: season and episode, matched as S01E02 or 1x02 in the name. */
  season?: number
  episode?: number
  /** Size bounds in bytes. */
  minSize?: number
  maxSize?: number
  sort?: SortOrder
  page?: number
  perPage?: number
  /** Overrides `page` (Newznab pages by offset). */
  offset?: number
  /** Count matches up to this many past the page; `total` is capped there. */
  countLimit?: number
}

/** A release in search results. */
export interface ReleaseSummary {
  id: number
  name: string
  source: string
  poster: string | null
  groups: string[]
  postedAt: Date | null
  fileTotal: number | null
  fileCount: number
  size: number
  readable: boolean
  complete: boolean | null
  category: number
  settled: boolean
}

const SORTS: Record<SortOrder, string> = {
  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
  largest: 'r.bytes DESC, r.id DESC',
  files: 'r.files DESC, r.id DESC',
  name: 'r.name ASC, r.id ASC',
}

/** A broad search needn't count every match (each counted row is a heap read). */
const COUNT_LIMIT = 10_000
const STATS_TTL_MS = 60 * 1000

/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
function contains(word: string): string {
  return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
}

/** Searching and browsing the catalog, for the web interface and Newznab. */
class SearchService {
  async search(params: SearchParams) {
    const where: string[] = ['r.files > 0']
    const bindings: unknown[] = []
    // Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike.
    for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
      where.push('r.name ILIKE ?')
      bindings.push(contains(word))
    }
    if (params.readable) where.push('r.readable')
    if (params.complete) where.push('r.complete')
    if (params.notIncomplete) where.push('r.complete IS NOT FALSE')
    if (params.source) {
      where.push('r.source = ?')
      bindings.push(params.source)
    }
    if (params.poster) {
      where.push('r.poster ILIKE ?')
      bindings.push(contains(params.poster))
    }
    if (params.categories?.length) {
      where.push('r.category = ANY(?)')
      bindings.push(params.categories)
    }
    if (params.maxAgeDays) {
      where.push('r.posted_at >= now() - make_interval(days => ?)')
      bindings.push(params.maxAgeDays)
    }
    if (params.minSize !== undefined) {
      where.push('r.bytes >= ?')
      bindings.push(params.minSize)
    }
    if (params.maxSize !== undefined) {
      where.push('r.bytes <= ?')
      bindings.push(params.maxSize)
    }
    if (params.season !== undefined) {
      const s = params.season
      const e = params.episode
      where.push('r.name ~* ?')
      bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
    }

    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
    const page = Math.max(params.page ?? 1, 1)
    const offset = params.offset ?? (page - 1) * perPage
    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
    const whereSql = where.join(' AND ')
    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
    const [{ rows }, { rows: counted }] = await Promise.all([
      db.rawQuery(
        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at
         FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`,
        [...bindings, perPage, offset]
      ),
      db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
    ])
    return {
      page,
      perPage,
      offset,
      total: Math.min(Number(counted[0].total), countLimit),
      /** More than `total` match (the count stops at `countLimit`). */
      totalCapped: Number(counted[0].total) > countLimit,
      releases: (rows as any[]).map((r): ReleaseSummary => ({
        id: Number(r.id),
        name: r.name,
        source: r.source,
        poster: r.poster,
        groups: r.groups,
        postedAt: r.posted_at,
        fileTotal: r.file_total,
        fileCount: Number(r.files),
        size: Number(r.bytes),
        readable: r.readable,
        complete: r.complete,
        category: r.category,
        settled: r.settled_at !== null,
      })),
    }
  }

  /** A release with its files, read from its chunks. */
  async release(id: number) {
    const r = await db.from('releases').where('id', id).first()
    if (!r) return null
    const files = await NzbService.files(id)
    return {
      id: Number(r.id),
      name: r.name,
      source: r.source,
      poster: r.poster,
      groups: r.groups,
      postedAt: r.posted_at,
      fileTotal: r.file_total,
      fileCount: Number(r.files),
      size: Number(r.bytes),
      segments: Number(r.segments),
      readable: r.readable,
      complete: r.complete,
      hasPar2: r.has_par2,
      category: r.category,
      firstSeenAt: r.first_seen_at,
      lastSeenAt: r.last_seen_at,
      settledAt: r.settled_at,
      files: files.map((f) => ({
        name: f.name,
        subject: f.subject,
        poster: f.poster,
        postedAt: f.postedAt === null ? null : new Date(f.postedAt),
        size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
        parts: f.segments.length,
      })),
    }
  }

  /** Totals for the web interface, computed at most once a minute (callers in between share them). */
  async stats() {
    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
      this.statsRefresh ??= this.computeStats()
        .then((value) => {
          this.statsCache = { at: Date.now(), value }
        })
        .catch((error) => {
          if (!this.statsCache) throw error
        })
        .finally(() => {
          this.statsRefresh = undefined
        })
      if (!this.statsCache) await this.statsRefresh
    }
    return this.statsCache!.value
  }

  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
  private statsRefresh?: Promise<void>

  /**
   * Estimates (a catalog of 100M+ rows can't be counted on every visit):
   * the row count from Postgres's statistics, below a million rows exact;
   * readable share, files and size from a sample; ranges counted exactly
   * (a few hundred thousand rows).
   */
  private async computeStats() {
    const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
    const estimate = Number(rows[0]?.n ?? 0)
    const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM (0.5)'
    const { rows: s } = await db.rawQuery(
      `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
       FROM releases ${sample} WHERE files > 0`
    )
    const sampled = Number(s[0].sampled)
    const releases = sample ? Math.max(estimate, 0) : sampled
    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
    const { rows: ranges } = await db.rawQuery(
      `SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total,
              count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history,
              max(done_at) AS last_done_at,
              max(done_at) FILTER (WHERE priority < 0) AS tail_done_at
       FROM ranges`
    )
    const g = ranges[0]
    return {
      approximate: sample !== '',
      releases,
      readableReleases: scale(s[0].readable),
      files: scale(s[0].files),
      size: scale(s[0].bytes),
      ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) },
      lastIndexedAt: g.last_done_at,
      tailIndexedAt: g.tail_done_at,
    }
  }
}

export default new SearchService()