usenet-indexer/app/services/search_service.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import NzbService from '#services/nzb_service'

export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'

/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
export interface SearchParams {
  q?: string
  /** Also releases the junk judge hid (release_verdicts). */
  judged?: boolean
  /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
  readable?: boolean
  /** Only releases known to have all their files. */
  complete?: boolean
  /** Leave out releases known to miss files (Newznab). */
  notIncomplete?: boolean
  /** Leave out releases the prober found passworded (encrypted archive entries or headers). */
  unlocked?: boolean
  /** How the release was recognised: stem, subject or post. */
  source?: string
  poster?: string
  /** Exact category ids (already expanded from parents). */
  categories?: number[]
  maxAgeDays?: number
  /** TV: season and episode, matched as S01E02 or 1x02 in the name. */
  season?: number
  episode?: number
  /** Size bounds in bytes. */
  minSize?: number
  maxSize?: number
  sort?: SortOrder
  page?: number
  perPage?: number
  /** Overrides `page` (Newznab pages by offset). */
  offset?: number
  /** Count matches up to this many past the page; `total` is capped there. */
  countLimit?: number
}

/** A release in search results. */
export interface ReleaseSummary {
  id: number
  name: string
  source: string
  poster: string | null
  groups: string[]
  postedAt: Date | null
  fileTotal: number | null
  fileCount: number
  size: number
  readable: boolean
  complete: boolean | null
  /** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
  final: boolean
  category: number
  settled: boolean
  /** The prober found encrypted archive entries or headers; null: not probed (yet). */
  passworded: boolean | null
  /** A password is known (release_passwords): it's in the NZB. */
  passwordKnown: boolean
  /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
  innerName: string | null
}

/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
const MAX_POSTERS = 200
const SORTS: Record<SortOrder, string> = {
  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
  largest: 'r.bytes DESC, r.id DESC',
  files: 'r.files DESC, r.id DESC',
  name: 'r.name ASC, r.id ASC',
}

/** A broad search needn't count every match (each counted row is a heap read). */
const COUNT_LIMIT = 10_000
const STATS_TTL_MS = 5 * 60 * 1000
/** A refresh that takes longer is given up (the next request tries again). */
const STATS_TIMEOUT_MS = 2 * 60 * 1000

/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
function contains(word: string): string {
  return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
}

/** Searching and browsing the catalog, for the web interface and Newznab. */
class SearchService {
  async search(params: SearchParams) {
    const where: string[] = ['r.files > 0']
    // Judged junk (app/services/release_judge.ts) is left out unless asked for.
    if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
    const bindings: unknown[] = []
    // Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
    for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
      where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
      bindings.push(contains(word), contains(word))
    }
    if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
    if (params.complete) where.push('r.complete')
    // Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
    if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
    if (params.unlocked) where.push('r.passworded IS NOT TRUE')
    if (params.source) {
      where.push('r.source = ?')
      bindings.push(params.source)
    }
    // A poster search: the matching posters first (posters, by trigram), each one's releases then by
    // releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
    // poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
    let posters: string[] | null = null
    if (params.poster) {
      const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
      posters = matched.map((m: { poster: string }) => m.poster)
    }
    if (params.categories?.length) {
      where.push('r.category = ANY(?)')
      bindings.push(params.categories)
    }
    if (params.maxAgeDays) {
      where.push('r.posted_at >= now() - make_interval(days => ?)')
      bindings.push(params.maxAgeDays)
    }
    if (params.minSize !== undefined) {
      where.push('r.bytes >= ?')
      bindings.push(params.minSize)
    }
    if (params.maxSize !== undefined) {
      where.push('r.bytes <= ?')
      bindings.push(params.maxSize)
    }
    if (params.season !== undefined) {
      const s = params.season
      const e = params.episode
      where.push('r.name ~* ?')
      bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
    }

    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
    const page = Math.max(params.page ?? 1, 1)
    const offset = params.offset ?? (page - 1) * perPage
    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
    const whereSql = where.join(' AND ')
    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
    if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
    // Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
    // bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
    const source =
      posters === null
        ? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
        : {
            from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
                     SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
            where: 'true',
            bindings: [posters] as unknown[],
          }
    const counting =
      posters === null
        ? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
        : db.rawQuery(
            `SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
               SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
            [posters, ...bindings]
          )
    const [{ rows }, { rows: counted }] = await Promise.all([
      db.rawQuery(
        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
                EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
                release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
         FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
        [...source.bindings, ...bindings, perPage, offset]
      ),
      counting,
    ])
    return {
      page,
      perPage,
      offset,
      total: Math.min(Number(counted[0].total), countLimit),
      /** More than `total` match (the count stops at `countLimit`). */
      totalCapped: Number(counted[0].total) > countLimit,
      releases: (rows as any[]).map((r): ReleaseSummary => ({
        id: Number(r.id),
        name: r.name,
        source: r.source,
        poster: r.poster,
        groups: r.groups,
        postedAt: r.posted_at,
        fileTotal: r.file_total,
        fileCount: Number(r.files),
        size: Number(r.bytes),
        readable: r.readable,
        complete: r.complete,
        final: r.final,
        category: r.category,
        settled: r.settled_at !== null,
        passworded: r.passworded,
        passwordKnown: r.password_known,
        innerName: r.inner_name,
      })),
    }
  }

  /** A release with its files, read from its chunks. */
  async release(id: number) {
    const r = await db.from('releases').where('id', id).first()
    if (!r) return null
    const [files, { rows: inspections }, { rows: finality }, passwords] = await Promise.all([
      NzbService.files(id),
      db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
      db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
      NzbService.passwords(id),
    ])
    return {
      id: Number(r.id),
      name: r.name,
      source: r.source,
      poster: r.poster,
      groups: r.groups,
      postedAt: r.posted_at,
      fileTotal: r.file_total,
      fileCount: Number(r.files),
      size: Number(r.bytes),
      segments: Number(r.segments),
      readable: r.readable,
      complete: r.complete,
      final: finality[0].final,
      hasPar2: r.has_par2,
      category: r.category,
      firstSeenAt: r.first_seen_at,
      lastSeenAt: r.last_seen_at,
      settledAt: r.settled_at,
      passworded: r.passworded,
      /** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
      passwords,
      innerName: r.inner_name,
      inspectedAt: r.inspected_at,
      /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
      inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
      files: files.map((f) => ({
        name: f.name,
        subject: f.subject,
        poster: f.poster,
        postedAt: f.postedAt === null ? null : new Date(f.postedAt),
        size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
        parts: f.segments.length,
      })),
    }
  }

  /** Totals for the web interface, computed at most once a minute (callers in between share them). */
  async stats() {
    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
      this.statsRefresh ??= Promise.race([
        this.computeStats(),
        new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
      ])
        .then((value) => {
          this.statsCache = { at: Date.now(), value }
        })
        .catch((error) => {
          logger.warn(`Stats: ${error.message}`)
          if (!this.statsCache) throw error
        })
        .finally(() => {
          this.statsRefresh = undefined
        })
      if (!this.statsCache) await this.statsRefresh
    }
    return this.statsCache!.value
  }

  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
  private statsRefresh?: Promise<void>

  /**
   * Estimates (a catalog of 100M+ rows can't be counted on every visit):
   * the row count from Postgres's statistics, below a million rows exact;
   * readable share, files and size from a sample of 5,000 rows; ranges
   * counted exactly (a few hundred thousand rows).
   */
  private async computeStats() {
    const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
    const estimate = Number(rows[0]?.n ?? 0)
    // A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
    const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
    const { rows: s } = await db.rawQuery(
      `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
       FROM releases ${sample} WHERE files > 0`
    )
    const sampled = Number(s[0].sampled)
    const releases = sample ? Math.max(estimate, 0) : sampled
    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
    // History by articles; the pass: the history ranges before those deferred behind them (priority
    // 2000 apart; e.g. groups added later that go first).
    const { rows: ranges } = await db.rawQuery(
      `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
            pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
       SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
              coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
              coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
              coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
              (SELECT max(done_at) FROM ranges) AS last_done_at,
              (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
       FROM h`
    )
    const g = ranges[0]
    return {
      approximate: sample !== '',
      releases,
      readableReleases: scale(s[0].readable),
      files: scale(s[0].files),
      size: scale(s[0].bytes),
      ranges: { done: Number(g.done), total: Number(g.total) },
      /** History and the current pass, in articles. */
      history: { done: Number(g.history_done), total: Number(g.history) },
      pass: { done: Number(g.pass_done), total: Number(g.pass) },
      lastIndexedAt: g.last_done_at,
      tailIndexedAt: g.tail_done_at,
    }
  }
}

export default new SearchService()