Rebuild: the catalog API and Newznab, from voyager 1

- /api (Newznab: caps, search, tvsearch, movie, music, book, get) for
  Prowlarr, Sonarr and Radarr: voyager 1's controller and XML on the new
  catalog; readable releases not known to miss files; NEWZNAB_API_KEY.
- /api/v1/releases (q, cat, readable, complete, source, poster, size,
  sort, paging), /api/v1/releases/:id with its files (decoded from the
  chunks; NzbService.files, shared with the NZB), /api/v1/stats
  (estimates, plus history and tail progress), /health. Open, as voyager
  1's catalog; the web interface uses it.
- release_category(name): voyager 1's categorizer without the archive
  contents, a generated column with a (category, posted_at) index.
- @adonisjs/static for the web interface (next).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/nzb_service.ts

@@ -1,31 +1,34 @@
 import db from '@adonisjs/lucid/services/db'
-import { decodeChunk } from '#lib/chunks/index'
+import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
 import { buildNzb } from '#lib/nzb/index'
 
 /**
- * A release's NZB (REBUILD.md, part 9), from its chunks in the chunk store:
- * every chunk decoded, its files in name order. A file stored twice (the same
- * name and first article, e.g. a range read again) is taken once.
+ * A release's files and NZB (REBUILD.md, part 9), from its chunks in the
+ * chunk store: every chunk decoded, its files in name order. A file stored
+ * twice (the same name and first article, e.g. a range read again) is taken
+ * once.
  */
 class NzbService {
-  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
-    const release = await db.from('releases').where('id', releaseId).select('name').first()
-    if (!release) return null
+  async files(releaseId: number): Promise<ChunkFile[]> {
     const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
-    const files = new Map<string, ReturnType<typeof decodeChunk>[number]>()
+    const files = new Map<string, ChunkFile>()
     for (const chunk of chunks) {
       for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
     }
+    return [...files.values()].sort((a, b) => a.name.localeCompare(b.name))
+  }
+
+  async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
+    const release = await db.from('releases').where('id', releaseId).select('name').first()
+    if (!release) return null
     const xml = buildNzb(
-      [...files.values()]
-        .sort((a, b) => a.name.localeCompare(b.name))
-        .map((f) => ({
-          name: f.name,
-          poster: f.poster,
-          date: f.postedAt ?? 0,
-          groups: f.groups,
-          segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
-        })),
+      (await this.files(releaseId)).map((f) => ({
+        name: f.name,
+        poster: f.poster,
+        date: f.postedAt ?? 0,
+        groups: f.groups,
+        segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
+      })),
       { title: release.name }
     )
     return { name: release.name, xml }

app/services/search_service.ts

@@ -0,0 +1,246 @@
+import db from '@adonisjs/lucid/services/db'
+import NzbService from '#services/nzb_service'
+
+export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
+
+/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
+export interface SearchParams {
+  q?: string
+  /** Only releases with a file whose name is readable (not an obfuscated token). */
+  readable?: boolean
+  /** Only releases known to have all their files. */
+  complete?: boolean
+  /** Leave out releases known to miss files (Newznab). */
+  notIncomplete?: boolean
+  /** How the release was recognised: stem, subject or post. */
+  source?: string
+  poster?: string
+  /** Exact category ids (already expanded from parents). */
+  categories?: number[]
+  maxAgeDays?: number
+  /** TV: season and episode, matched as S01E02 or 1x02 in the name. */
+  season?: number
+  episode?: number
+  /** Size bounds in bytes. */
+  minSize?: number
+  maxSize?: number
+  sort?: SortOrder
+  page?: number
+  perPage?: number
+  /** Overrides `page` (Newznab pages by offset). */
+  offset?: number
+  /** Count matches up to this many past the page; `total` is capped there. */
+  countLimit?: number
+}
+
+/** A release in search results. */
+export interface ReleaseSummary {
+  id: number
+  name: string
+  source: string
+  poster: string | null
+  groups: string[]
+  postedAt: Date | null
+  fileTotal: number | null
+  fileCount: number
+  size: number
+  readable: boolean
+  complete: boolean | null
+  category: number
+  settled: boolean
+}
+
+const SORTS: Record<SortOrder, string> = {
+  newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
+  oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
+  largest: 'r.bytes DESC, r.id DESC',
+  files: 'r.files DESC, r.id DESC',
+  name: 'r.name ASC, r.id ASC',
+}
+
+/** A broad search needn't count every match (each counted row is a heap read). */
+const COUNT_LIMIT = 10_000
+const STATS_TTL_MS = 60 * 1000
+
+/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
+function contains(word: string): string {
+  return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
+}
+
+/** Searching and browsing the catalog, for the web interface and Newznab. */
+class SearchService {
+  async search(params: SearchParams) {
+    const where: string[] = ['r.files > 0']
+    const bindings: unknown[] = []
+    // Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike.
+    for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
+      where.push('r.name ILIKE ?')
+      bindings.push(contains(word))
+    }
+    if (params.readable) where.push('r.readable')
+    if (params.complete) where.push('r.complete')
+    if (params.notIncomplete) where.push('r.complete IS NOT FALSE')
+    if (params.source) {
+      where.push('r.source = ?')
+      bindings.push(params.source)
+    }
+    if (params.poster) {
+      where.push('r.poster ILIKE ?')
+      bindings.push(contains(params.poster))
+    }
+    if (params.categories?.length) {
+      where.push('r.category = ANY(?)')
+      bindings.push(params.categories)
+    }
+    if (params.maxAgeDays) {
+      where.push('r.posted_at >= now() - make_interval(days => ?)')
+      bindings.push(params.maxAgeDays)
+    }
+    if (params.minSize !== undefined) {
+      where.push('r.bytes >= ?')
+      bindings.push(params.minSize)
+    }
+    if (params.maxSize !== undefined) {
+      where.push('r.bytes <= ?')
+      bindings.push(params.maxSize)
+    }
+    if (params.season !== undefined) {
+      const s = params.season
+      const e = params.episode
+      where.push('r.name ~* ?')
+      bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
+    }
+
+    const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
+    const page = Math.max(params.page ?? 1, 1)
+    const offset = params.offset ?? (page - 1) * perPage
+    const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
+    const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
+    const [{ rows }, { rows: counted }] = await Promise.all([
+      db.rawQuery(
+        `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at
+         FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`,
+        [...bindings, perPage, offset]
+      ),
+      db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
+    ])
+    return {
+      page,
+      perPage,
+      offset,
+      total: Math.min(Number(counted[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(counted[0].total) > countLimit,
+      releases: (rows as any[]).map((r): ReleaseSummary => ({
+        id: Number(r.id),
+        name: r.name,
+        source: r.source,
+        poster: r.poster,
+        groups: r.groups,
+        postedAt: r.posted_at,
+        fileTotal: r.file_total,
+        fileCount: Number(r.files),
+        size: Number(r.bytes),
+        readable: r.readable,
+        complete: r.complete,
+        category: r.category,
+        settled: r.settled_at !== null,
+      })),
+    }
+  }
+
+  /** A release with its files, read from its chunks. */
+  async release(id: number) {
+    const r = await db.from('releases').where('id', id).first()
+    if (!r) return null
+    const files = await NzbService.files(id)
+    return {
+      id: Number(r.id),
+      name: r.name,
+      source: r.source,
+      poster: r.poster,
+      groups: r.groups,
+      postedAt: r.posted_at,
+      fileTotal: r.file_total,
+      fileCount: Number(r.files),
+      size: Number(r.bytes),
+      segments: Number(r.segments),
+      readable: r.readable,
+      complete: r.complete,
+      hasPar2: r.has_par2,
+      category: r.category,
+      firstSeenAt: r.first_seen_at,
+      lastSeenAt: r.last_seen_at,
+      settledAt: r.settled_at,
+      files: files.map((f) => ({
+        name: f.name,
+        subject: f.subject,
+        poster: f.poster,
+        postedAt: f.postedAt === null ? null : new Date(f.postedAt),
+        size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
+        parts: f.segments.length,
+      })),
+    }
+  }
+
+  /** Totals for the web interface, computed at most once a minute (callers in between share them). */
+  async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= this.computeStats()
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        .catch((error) => {
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates (a catalog of 100M+ rows can't be counted on every visit):
+   * the row count from Postgres's statistics, below a million rows exact;
+   * readable share, files and size from a sample; ranges counted exactly
+   * (a few hundred thousand rows).
+   */
+  private async computeStats() {
+    const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
+    const estimate = Number(rows[0]?.n ?? 0)
+    const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM (0.5)'
+    const { rows: s } = await db.rawQuery(
+      `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
+       FROM releases ${sample} WHERE files > 0`
+    )
+    const sampled = Number(s[0].sampled)
+    const releases = sample ? Math.max(estimate, 0) : sampled
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
+    const { rows: ranges } = await db.rawQuery(
+      `SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total,
+              count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history,
+              max(done_at) AS last_done_at,
+              max(done_at) FILTER (WHERE priority < 0) AS tail_done_at
+       FROM ranges`
+    )
+    const g = ranges[0]
+    return {
+      approximate: sample !== '',
+      releases,
+      readableReleases: scale(s[0].readable),
+      files: scale(s[0].files),
+      size: scale(s[0].bytes),
+      ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) },
+      lastIndexedAt: g.last_done_at,
+      tailIndexedAt: g.tail_done_at,
+    }
+  }
+}
+
+export default new SearchService()