318 lines
14 KiB
TypeScript
318 lines
14 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import NzbService from '#services/nzb_service'
|
|
|
|
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
|
|
|
/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
|
|
export interface SearchParams {
|
|
q?: string
|
|
/** Also releases the junk judge hid (release_verdicts). */
|
|
judged?: boolean
|
|
/** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
|
|
readable?: boolean
|
|
/** Only releases known to have all their files. */
|
|
complete?: boolean
|
|
/** Leave out releases known to miss files (Newznab). */
|
|
notIncomplete?: boolean
|
|
/** Leave out releases the prober found passworded (encrypted archive entries or headers). */
|
|
unlocked?: boolean
|
|
/** How the release was recognised: stem, subject or post. */
|
|
source?: string
|
|
poster?: string
|
|
/** Exact category ids (already expanded from parents). */
|
|
categories?: number[]
|
|
maxAgeDays?: number
|
|
/** TV: season and episode, matched as S01E02 or 1x02 in the name. */
|
|
season?: number
|
|
episode?: number
|
|
/** Size bounds in bytes. */
|
|
minSize?: number
|
|
maxSize?: number
|
|
sort?: SortOrder
|
|
page?: number
|
|
perPage?: number
|
|
/** Overrides `page` (Newznab pages by offset). */
|
|
offset?: number
|
|
/** Count matches up to this many past the page; `total` is capped there. */
|
|
countLimit?: number
|
|
}
|
|
|
|
/** A release in search results. */
|
|
export interface ReleaseSummary {
|
|
id: number
|
|
name: string
|
|
source: string
|
|
poster: string | null
|
|
groups: string[]
|
|
postedAt: Date | null
|
|
fileTotal: number | null
|
|
fileCount: number
|
|
size: number
|
|
readable: boolean
|
|
complete: boolean | null
|
|
/** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
|
|
final: boolean
|
|
category: number
|
|
settled: boolean
|
|
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
|
passworded: boolean | null
|
|
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
|
|
innerName: string | null
|
|
}
|
|
|
|
/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
|
|
const MAX_POSTERS = 200
|
|
const SORTS: Record<SortOrder, string> = {
|
|
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
|
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
|
largest: 'r.bytes DESC, r.id DESC',
|
|
files: 'r.files DESC, r.id DESC',
|
|
name: 'r.name ASC, r.id ASC',
|
|
}
|
|
|
|
/** A broad search needn't count every match (each counted row is a heap read). */
|
|
const COUNT_LIMIT = 10_000
|
|
const STATS_TTL_MS = 5 * 60 * 1000
|
|
/** A refresh that takes longer is given up (the next request tries again). */
|
|
const STATS_TIMEOUT_MS = 2 * 60 * 1000
|
|
|
|
/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
|
|
function contains(word: string): string {
|
|
return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
|
|
}
|
|
|
|
/** Searching and browsing the catalog, for the web interface and Newznab. */
|
|
class SearchService {
|
|
async search(params: SearchParams) {
|
|
const where: string[] = ['r.files > 0']
|
|
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
|
|
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
|
const bindings: unknown[] = []
|
|
// Every word in the name or the inner name the prober found, through their trigram indexes; "The.Block.AU" and "the block au" alike.
|
|
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
|
where.push('(r.name ILIKE ? OR r.inner_name ILIKE ?)')
|
|
bindings.push(contains(word), contains(word))
|
|
}
|
|
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
|
if (params.complete) where.push('r.complete')
|
|
// Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
|
|
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
|
|
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
|
if (params.source) {
|
|
where.push('r.source = ?')
|
|
bindings.push(params.source)
|
|
}
|
|
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
|
|
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
|
|
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
|
|
let posters: string[] | null = null
|
|
if (params.poster) {
|
|
const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
|
posters = matched.map((m: { poster: string }) => m.poster)
|
|
}
|
|
if (params.categories?.length) {
|
|
where.push('r.category = ANY(?)')
|
|
bindings.push(params.categories)
|
|
}
|
|
if (params.maxAgeDays) {
|
|
where.push('r.posted_at >= now() - make_interval(days => ?)')
|
|
bindings.push(params.maxAgeDays)
|
|
}
|
|
if (params.minSize !== undefined) {
|
|
where.push('r.bytes >= ?')
|
|
bindings.push(params.minSize)
|
|
}
|
|
if (params.maxSize !== undefined) {
|
|
where.push('r.bytes <= ?')
|
|
bindings.push(params.maxSize)
|
|
}
|
|
if (params.season !== undefined) {
|
|
const s = params.season
|
|
const e = params.episode
|
|
where.push('r.name ~* ?')
|
|
bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
|
|
}
|
|
|
|
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
|
const page = Math.max(params.page ?? 1, 1)
|
|
const offset = params.offset ?? (page - 1) * perPage
|
|
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
|
const whereSql = where.join(' AND ')
|
|
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
|
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
|
|
// Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
|
|
// bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
|
|
const source =
|
|
posters === null
|
|
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
|
|
: {
|
|
from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
|
SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
|
where: 'true',
|
|
bindings: [posters] as unknown[],
|
|
}
|
|
const counting =
|
|
posters === null
|
|
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
|
|
: db.rawQuery(
|
|
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
|
SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
|
[posters, ...bindings]
|
|
)
|
|
const [{ rows }, { rows: counted }] = await Promise.all([
|
|
db.rawQuery(
|
|
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
|
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
|
|
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
|
[...source.bindings, ...bindings, perPage, offset]
|
|
),
|
|
counting,
|
|
])
|
|
return {
|
|
page,
|
|
perPage,
|
|
offset,
|
|
total: Math.min(Number(counted[0].total), countLimit),
|
|
/** More than `total` match (the count stops at `countLimit`). */
|
|
totalCapped: Number(counted[0].total) > countLimit,
|
|
releases: (rows as any[]).map((r): ReleaseSummary => ({
|
|
id: Number(r.id),
|
|
name: r.name,
|
|
source: r.source,
|
|
poster: r.poster,
|
|
groups: r.groups,
|
|
postedAt: r.posted_at,
|
|
fileTotal: r.file_total,
|
|
fileCount: Number(r.files),
|
|
size: Number(r.bytes),
|
|
readable: r.readable,
|
|
complete: r.complete,
|
|
final: r.final,
|
|
category: r.category,
|
|
settled: r.settled_at !== null,
|
|
passworded: r.passworded,
|
|
innerName: r.inner_name,
|
|
})),
|
|
}
|
|
}
|
|
|
|
/** A release with its files, read from its chunks. */
|
|
async release(id: number) {
|
|
const r = await db.from('releases').where('id', id).first()
|
|
if (!r) return null
|
|
const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
|
|
NzbService.files(id),
|
|
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
|
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
|
|
])
|
|
return {
|
|
id: Number(r.id),
|
|
name: r.name,
|
|
source: r.source,
|
|
poster: r.poster,
|
|
groups: r.groups,
|
|
postedAt: r.posted_at,
|
|
fileTotal: r.file_total,
|
|
fileCount: Number(r.files),
|
|
size: Number(r.bytes),
|
|
segments: Number(r.segments),
|
|
readable: r.readable,
|
|
complete: r.complete,
|
|
final: finality[0].final,
|
|
hasPar2: r.has_par2,
|
|
category: r.category,
|
|
firstSeenAt: r.first_seen_at,
|
|
lastSeenAt: r.last_seen_at,
|
|
settledAt: r.settled_at,
|
|
passworded: r.passworded,
|
|
innerName: r.inner_name,
|
|
inspectedAt: r.inspected_at,
|
|
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
|
|
inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
|
|
files: files.map((f) => ({
|
|
name: f.name,
|
|
subject: f.subject,
|
|
poster: f.poster,
|
|
postedAt: f.postedAt === null ? null : new Date(f.postedAt),
|
|
size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
|
|
parts: f.segments.length,
|
|
})),
|
|
}
|
|
}
|
|
|
|
/** Totals for the web interface, computed at most once a minute (callers in between share them). */
|
|
async stats() {
|
|
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
|
this.statsRefresh ??= Promise.race([
|
|
this.computeStats(),
|
|
new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
|
|
])
|
|
.then((value) => {
|
|
this.statsCache = { at: Date.now(), value }
|
|
})
|
|
.catch((error) => {
|
|
logger.warn(`Stats: ${error.message}`)
|
|
if (!this.statsCache) throw error
|
|
})
|
|
.finally(() => {
|
|
this.statsRefresh = undefined
|
|
})
|
|
if (!this.statsCache) await this.statsRefresh
|
|
}
|
|
return this.statsCache!.value
|
|
}
|
|
|
|
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
|
private statsRefresh?: Promise<void>
|
|
|
|
/**
|
|
* Estimates (a catalog of 100M+ rows can't be counted on every visit):
|
|
* the row count from Postgres's statistics, below a million rows exact;
|
|
* readable share, files and size from a sample of 5,000 rows; ranges
|
|
* counted exactly (a few hundred thousand rows).
|
|
*/
|
|
private async computeStats() {
|
|
const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
|
|
const estimate = Number(rows[0]?.n ?? 0)
|
|
// A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
|
|
const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
|
|
const { rows: s } = await db.rawQuery(
|
|
`SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
|
|
FROM releases ${sample} WHERE files > 0`
|
|
)
|
|
const sampled = Number(s[0].sampled)
|
|
const releases = sample ? Math.max(estimate, 0) : sampled
|
|
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
|
// History by articles; the pass: the history ranges before those deferred behind them (priority
|
|
// 2000 apart; e.g. groups added later that go first).
|
|
const { rows: ranges } = await db.rawQuery(
|
|
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0),
|
|
pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
|
|
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges) AS total,
|
|
coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
|
|
coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
|
|
coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
|
|
(SELECT max(done_at) FROM ranges) AS last_done_at,
|
|
(SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
|
|
FROM h`
|
|
)
|
|
const g = ranges[0]
|
|
return {
|
|
approximate: sample !== '',
|
|
releases,
|
|
readableReleases: scale(s[0].readable),
|
|
files: scale(s[0].files),
|
|
size: scale(s[0].bytes),
|
|
ranges: { done: Number(g.done), total: Number(g.total) },
|
|
/** History and the current pass, in articles. */
|
|
history: { done: Number(g.history_done), total: Number(g.history) },
|
|
pass: { done: Number(g.pass_done), total: Number(g.pass) },
|
|
lastIndexedAt: g.last_done_at,
|
|
tailIndexedAt: g.tail_done_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
export default new SearchService()
|