import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import NzbService from '#services/nzb_service' export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name' /** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */ export interface SearchParams { q?: string /** Only releases with a file whose name is readable (not an obfuscated token). */ readable?: boolean /** Only releases known to have all their files. */ complete?: boolean /** Leave out releases known to miss files (Newznab). */ notIncomplete?: boolean /** How the release was recognised: stem, subject or post. */ source?: string poster?: string /** Exact category ids (already expanded from parents). */ categories?: number[] maxAgeDays?: number /** TV: season and episode, matched as S01E02 or 1x02 in the name. */ season?: number episode?: number /** Size bounds in bytes. */ minSize?: number maxSize?: number sort?: SortOrder page?: number perPage?: number /** Overrides `page` (Newznab pages by offset). */ offset?: number /** Count matches up to this many past the page; `total` is capped there. */ countLimit?: number } /** A release in search results. */ export interface ReleaseSummary { id: number name: string source: string poster: string | null groups: string[] postedAt: Date | null fileTotal: number | null fileCount: number size: number readable: boolean complete: boolean | null category: number settled: boolean } const SORTS: Record = { newest: 'r.posted_at DESC NULLS LAST, r.id DESC', oldest: 'r.posted_at ASC NULLS LAST, r.id ASC', largest: 'r.bytes DESC, r.id DESC', files: 'r.files DESC, r.id DESC', name: 'r.name ASC, r.id ASC', } /** A broad search needn't count every match (each counted row is a heap read). */ const COUNT_LIMIT = 10_000 const STATS_TTL_MS = 5 * 60 * 1000 /** A refresh that takes longer is given up (the next request tries again). */ const STATS_TIMEOUT_MS = 2 * 60 * 1000 /** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */ function contains(word: string): string { return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%` } /** Searching and browsing the catalog, for the web interface and Newznab. */ class SearchService { async search(params: SearchParams) { const where: string[] = ['r.files > 0'] const bindings: unknown[] = [] // Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike. for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) { where.push('r.name ILIKE ?') bindings.push(contains(word)) } if (params.readable) where.push('r.readable') if (params.complete) where.push('r.complete') if (params.notIncomplete) where.push('r.complete IS NOT FALSE') if (params.source) { where.push('r.source = ?') bindings.push(params.source) } if (params.poster) { where.push('r.poster ILIKE ?') bindings.push(contains(params.poster)) } if (params.categories?.length) { where.push('r.category = ANY(?)') bindings.push(params.categories) } if (params.maxAgeDays) { where.push('r.posted_at >= now() - make_interval(days => ?)') bindings.push(params.maxAgeDays) } if (params.minSize !== undefined) { where.push('r.bytes >= ?') bindings.push(params.minSize) } if (params.maxSize !== undefined) { where.push('r.bytes <= ?') bindings.push(params.maxSize) } if (params.season !== undefined) { const s = params.season const e = params.episode where.push('r.name ~* ?') bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`) } const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200) const page = Math.max(params.page ?? 1, 1) const offset = params.offset ?? (page - 1) * perPage const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest const whereSql = where.join(' AND ') const countLimit = offset + (params.countLimit ?? COUNT_LIMIT) const [{ rows }, { rows: counted }] = await Promise.all([ db.rawQuery( `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`, [...bindings, perPage, offset] ), db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings), ]) return { page, perPage, offset, total: Math.min(Number(counted[0].total), countLimit), /** More than `total` match (the count stops at `countLimit`). */ totalCapped: Number(counted[0].total) > countLimit, releases: (rows as any[]).map((r): ReleaseSummary => ({ id: Number(r.id), name: r.name, source: r.source, poster: r.poster, groups: r.groups, postedAt: r.posted_at, fileTotal: r.file_total, fileCount: Number(r.files), size: Number(r.bytes), readable: r.readable, complete: r.complete, category: r.category, settled: r.settled_at !== null, })), } } /** A release with its files, read from its chunks. */ async release(id: number) { const r = await db.from('releases').where('id', id).first() if (!r) return null const files = await NzbService.files(id) return { id: Number(r.id), name: r.name, source: r.source, poster: r.poster, groups: r.groups, postedAt: r.posted_at, fileTotal: r.file_total, fileCount: Number(r.files), size: Number(r.bytes), segments: Number(r.segments), readable: r.readable, complete: r.complete, hasPar2: r.has_par2, category: r.category, firstSeenAt: r.first_seen_at, lastSeenAt: r.last_seen_at, settledAt: r.settled_at, files: files.map((f) => ({ name: f.name, subject: f.subject, poster: f.poster, postedAt: f.postedAt === null ? null : new Date(f.postedAt), size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0), parts: f.segments.length, })), } } /** Totals for the web interface, computed at most once a minute (callers in between share them). */ async stats() { if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) { this.statsRefresh ??= Promise.race([ this.computeStats(), new Promise((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()), ]) .then((value) => { this.statsCache = { at: Date.now(), value } }) .catch((error) => { logger.warn(`Stats: ${error.message}`) if (!this.statsCache) throw error }) .finally(() => { this.statsRefresh = undefined }) if (!this.statsCache) await this.statsRefresh } return this.statsCache!.value } private statsCache?: { at: number; value: Awaited> } private statsRefresh?: Promise /** * Estimates (a catalog of 100M+ rows can't be counted on every visit): * the row count from Postgres's statistics, below a million rows exact; * readable share, files and size from a sample of 5,000 rows; ranges * counted exactly (a few hundred thousand rows). */ private async computeStats() { const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`) const estimate = Number(rows[0]?.n ?? 0) // A fixed-size sample: a percentage of a big catalog is many random reads on the HDD. const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)' const { rows: s } = await db.rawQuery( `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes FROM releases ${sample} WHERE files > 0` ) const sampled = Number(s[0].sampled) const releases = sample ? Math.max(estimate, 0) : sampled const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0) // History by articles; the pass: the history ranges before those deferred behind them (priority // 2000 apart; e.g. groups added later that go first). const { rows: ranges } = await db.rawQuery( `WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0), pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h) SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total, coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history, coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done, coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass, (SELECT max(done_at) FROM ranges) AS last_done_at, (SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at FROM h` ) const g = ranges[0] return { approximate: sample !== '', releases, readableReleases: scale(s[0].readable), files: scale(s[0].files), size: scale(s[0].bytes), ranges: { done: Number(g.done), total: Number(g.total) }, /** History and the current pass, in articles. */ history: { done: Number(g.history_done), total: Number(g.history) }, pass: { done: Number(g.pass_done), total: Number(g.pass) }, lastIndexedAt: g.last_done_at, tailIndexedAt: g.tail_done_at, } } } export default new SearchService()