import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import NzbService from '#services/nzb_service' import { SOURCES, sourceName } from '#lib/release/source' export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name' /** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */ export interface SearchParams { q?: string /** Also releases the junk judge hid (release_verdicts). */ judged?: boolean /** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */ readable?: boolean /** Only releases known to have all their files. */ complete?: boolean /** Leave out releases known to miss files (Newznab). */ notIncomplete?: boolean /** Leave out releases the prober found passworded (encrypted archive entries or headers). */ unlocked?: boolean /** How the release was recognised: stem, subject or post. */ source?: string poster?: string /** Exact category ids (already expanded from parents). */ categories?: number[] maxAgeDays?: number /** TV: season and episode, matched as S01E02 or 1x02 in the name. */ season?: number episode?: number /** Size bounds in bytes. */ minSize?: number maxSize?: number sort?: SortOrder page?: number perPage?: number /** Overrides `page` (Newznab pages by offset). */ offset?: number /** Count matches up to this many past the page; `total` is capped there. */ countLimit?: number } /** A release in search results. */ export interface ReleaseSummary { id: number name: string source: string poster: string | null groups: string[] postedAt: Date | null fileTotal: number | null fileCount: number size: number readable: boolean complete: boolean | null /** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */ final: boolean category: number settled: boolean /** The prober found encrypted archive entries or headers; null: not probed (yet). */ passworded: boolean | null /** A password is known (release_passwords): it's in the NZB. */ passwordKnown: boolean /** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */ innerName: string | null } /** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */ const MAX_POSTERS = 200 const SORTS: Record = { newest: 'r.posted_at DESC NULLS LAST, r.id DESC', oldest: 'r.posted_at ASC NULLS LAST, r.id ASC', largest: 'r.bytes DESC, r.id DESC', files: 'r.files DESC, r.id DESC', name: 'r.name ASC, r.id ASC', } /** A broad search needn't count every match (each counted row is a heap read). */ const COUNT_LIMIT = 10_000 const STATS_TTL_MS = 5 * 60 * 1000 /** A refresh that takes longer is given up (the next request tries again). */ const STATS_TIMEOUT_MS = 2 * 60 * 1000 /** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */ function contains(word: string): string { return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%` } /** Searching and browsing the catalog, for the web interface and Newznab. */ class SearchService { async search(params: SearchParams) { const where: string[] = ['r.files > 0'] // Judged junk (app/services/release_judge.ts) is left out unless asked for. if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)') const bindings: unknown[] = [] // Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its // trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike. for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) { where.push('r.id IN (SELECT t.release_id FROM release_texts t WHERE t.text ILIKE ?)') bindings.push(contains(word)) } if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)') if (params.complete) where.push('r.complete') // Incomplete only when final: a provisional tail release may yet get its missing files. if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)') if (params.unlocked) where.push('r.passworded IS NOT TRUE') if (params.source) { const code = (SOURCES as Record)[params.source] if (code === undefined) where.push('false') else { where.push('r.source = ?') bindings.push(code) } } // A poster search: the matching posters first (posters, by trigram), each one's releases then by // releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a // poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)). let posters: number[] | null = null if (params.poster) { const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)]) posters = matched.map((m: { id: string }) => Number(m.id)) } if (params.categories?.length) { where.push('r.category = ANY(?)') bindings.push(params.categories) } if (params.maxAgeDays) { where.push('r.posted_at >= now() - make_interval(days => ?)') bindings.push(params.maxAgeDays) } if (params.minSize !== undefined) { where.push('r.bytes >= ?') bindings.push(params.minSize) } if (params.maxSize !== undefined) { where.push('r.bytes <= ?') bindings.push(params.maxSize) } if (params.season !== undefined) { const s = params.season const e = params.episode where.push('r.name ~* ?') bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`) } const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200) const page = Math.max(params.page ?? 1, 1) const offset = params.offset ?? (page - 1) * perPage const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest const whereSql = where.join(' AND ') const countLimit = offset + (params.countLimit ?? COUNT_LIMIT) if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] } // Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage // in the order asked; then merged. const source = posters === null ? { from: 'releases r', where: whereSql, bindings: [] as unknown[] } : { from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL ( SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`, where: 'true', bindings: [posters] as unknown[], } const counting = posters === null ? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings) : db.rawQuery( `SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL ( SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`, [posters, ...bindings] ) const [{ rows }, { rows: counted }] = await Promise.all([ db.rawQuery( `SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name, EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known, r.settled_at IS NOT NULL AS final FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`, [...source.bindings, ...bindings, perPage, offset] ), counting, ]) return { page, perPage, offset, total: Math.min(Number(counted[0].total), countLimit), /** More than `total` match (the count stops at `countLimit`). */ totalCapped: Number(counted[0].total) > countLimit, releases: (rows as any[]).map((r): ReleaseSummary => ({ id: Number(r.id), name: r.name, source: sourceName(r.source), poster: r.poster ?? null, groups: r.groups ?? [], postedAt: r.posted_at, fileTotal: r.file_total, fileCount: Number(r.files), size: Number(r.bytes), readable: r.readable, complete: r.complete, final: r.final, category: r.category, settled: r.settled_at !== null, passworded: r.passworded, passwordKnown: r.password_known, innerName: r.inner_name, })), } } /** A release with its files, read from its chunks. */ async release(id: number) { const { rows: [r], } = await db.rawQuery( `SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`, [id] ) if (!r) return null const [files, { rows: inspections }, passwords] = await Promise.all([ NzbService.files(id), db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]), NzbService.passwords(id), ]) return { id: Number(r.id), name: r.name, source: sourceName(r.source), poster: r.poster ?? null, groups: r.groups ?? [], postedAt: r.posted_at, fileTotal: r.file_total, fileCount: Number(r.files), size: Number(r.bytes), segments: Number(r.segments), readable: r.readable, complete: r.complete, final: r.settled_at !== null, hasPar2: r.has_par2, category: r.category, firstSeenAt: r.first_seen_at, settledAt: r.settled_at, passworded: r.passworded, /** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */ passwords, innerName: r.inner_name, inspectedAt: r.inspected_at, /** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */ inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })), files: files.map((f) => ({ name: f.name, subject: f.subject, poster: f.poster, postedAt: f.postedAt === null ? null : new Date(f.postedAt), size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0), parts: f.segments.length, })), } } /** Totals for the web interface, computed at most once a minute (callers in between share them). */ async stats() { if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) { this.statsRefresh ??= Promise.race([ this.computeStats(), new Promise((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()), ]) .then((value) => { this.statsCache = { at: Date.now(), value } }) .catch((error) => { logger.warn(`Stats: ${error.message}`) if (!this.statsCache) throw error }) .finally(() => { this.statsRefresh = undefined }) if (!this.statsCache) await this.statsRefresh } return this.statsCache!.value } private statsCache?: { at: number; value: Awaited> } private statsRefresh?: Promise /** * Estimates (a catalog of 100M+ rows can't be counted on every visit): * the row count from Postgres's statistics, below a million rows exact; * readable share, files and size from a sample of 5,000 rows; ranges * counted exactly (a few hundred thousand rows). */ private async computeStats() { const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`) const estimate = Number(rows[0]?.n ?? 0) // A fixed-size sample: a percentage of a big catalog is many random reads on the HDD. const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)' const { rows: s } = await db.rawQuery( `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes FROM releases ${sample} WHERE files > 0` ) const sampled = Number(s[0].sampled) const releases = sample ? Math.max(estimate, 0) : sampled const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0) // History by articles; the pass: the history ranges before those deferred behind them (priority // 2000 apart; e.g. groups added later that go first). const { rows: ranges } = await db.rawQuery( `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0), pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h) SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total, coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history, coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done, coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass, (SELECT max(done_at) FROM ranges) AS last_done_at, (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at FROM h` ) const g = ranges[0] return { approximate: sample !== '', releases, readableReleases: scale(s[0].readable), files: scale(s[0].files), size: scale(s[0].bytes), ranges: { done: Number(g.done), total: Number(g.total) }, /** History and the current pass, in articles. */ history: { done: Number(g.history_done), total: Number(g.history) }, pass: { done: Number(g.pass_done), total: Number(g.pass) }, lastIndexedAt: g.last_done_at, tailIndexedAt: g.tail_done_at, } } } export default new SearchService()