Daan MeijerandClaude Opus 5.5
5132ab9083
"^zed" finds "Zed Dead", "The_Zed_Files", not "Facialized": a regex on release_texts, (^|[^[:alnum:]]) before the word, which the trigram index serves (homelab: 551 texts for ^zed against 6,606 for zed, 35 ms warm). Words without ^ match anywhere as before; the search box says so. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
350 lines
15 KiB
TypeScript
350 lines
15 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import NzbService from '#services/nzb_service'
|
|
import { SOURCES, sourceName } from '#lib/release/source'
|
|
|
|
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
|
|
|
/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
|
|
export interface SearchParams {
|
|
q?: string
|
|
/** Also releases the junk judge hid (release_verdicts). */
|
|
judged?: boolean
|
|
/** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
|
|
readable?: boolean
|
|
/** Only releases known to have all their files. */
|
|
complete?: boolean
|
|
/** Leave out releases known to miss files (Newznab). */
|
|
notIncomplete?: boolean
|
|
/** Leave out releases the prober found passworded (encrypted archive entries or headers). */
|
|
unlocked?: boolean
|
|
/** How the release was recognised: stem, subject or post. */
|
|
source?: string
|
|
poster?: string
|
|
/** Exact category ids (already expanded from parents). */
|
|
categories?: number[]
|
|
maxAgeDays?: number
|
|
/** TV: season and episode, matched as S01E02 or 1x02 in the name. */
|
|
season?: number
|
|
episode?: number
|
|
/** Size bounds in bytes. */
|
|
minSize?: number
|
|
maxSize?: number
|
|
sort?: SortOrder
|
|
page?: number
|
|
perPage?: number
|
|
/** Overrides `page` (Newznab pages by offset). */
|
|
offset?: number
|
|
/** Count matches up to this many past the page; `total` is capped there. */
|
|
countLimit?: number
|
|
}
|
|
|
|
/** A release in search results. */
|
|
export interface ReleaseSummary {
|
|
id: number
|
|
name: string
|
|
source: string
|
|
poster: string | null
|
|
groups: string[]
|
|
postedAt: Date | null
|
|
fileTotal: number | null
|
|
fileCount: number
|
|
size: number
|
|
readable: boolean
|
|
complete: boolean | null
|
|
/** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
|
|
final: boolean
|
|
category: number
|
|
settled: boolean
|
|
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
|
passworded: boolean | null
|
|
/** A password is known (release_passwords): it's in the NZB. */
|
|
passwordKnown: boolean
|
|
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
|
|
innerName: string | null
|
|
}
|
|
|
|
/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
|
|
const MAX_POSTERS = 200
|
|
const SORTS: Record<SortOrder, string> = {
|
|
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
|
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
|
largest: 'r.bytes DESC, r.id DESC',
|
|
files: 'r.files DESC, r.id DESC',
|
|
name: 'r.name ASC, r.id ASC',
|
|
}
|
|
|
|
/** A broad search needn't count every match (each counted row is a heap read). */
|
|
const COUNT_LIMIT = 10_000
|
|
const STATS_TTL_MS = 5 * 60 * 1000
|
|
/** A refresh that takes longer is given up (the next request tries again). */
|
|
const STATS_TIMEOUT_MS = 2 * 60 * 1000
|
|
|
|
/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
|
|
function contains(word: string): string {
|
|
return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
|
|
}
|
|
|
|
/**
|
|
* A search word's condition on release_texts: anywhere in a text (ILIKE), or with a leading "^" at a word's start, the
|
|
* text's or after a character that's no letter or digit ("^zed" finds "Zed.Dead", "The_Zed", not "Facialized"). Both
|
|
* use the trigram index.
|
|
*/
|
|
function wordMatch(word: string): { sql: string; binding: string } {
|
|
if (word.startsWith('^') && word.length > 1) {
|
|
const rest = word.slice(1).replace(/[\\^$.|?*+()[\]{}]/g, (c) => `\\${c}`)
|
|
return { sql: 't.text ~* ?', binding: `(^|[^[:alnum:]])${rest}` }
|
|
}
|
|
return { sql: 't.text ILIKE ?', binding: contains(word) }
|
|
}
|
|
|
|
/** Searching and browsing the catalog, for the web interface and Newznab. */
|
|
class SearchService {
|
|
async search(params: SearchParams) {
|
|
const where: string[] = ['r.files > 0']
|
|
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
|
|
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
|
const bindings: unknown[] = []
|
|
// Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
|
|
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike;
|
|
// "^zed" only at a word's start.
|
|
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
|
const match = wordMatch(word)
|
|
where.push(`r.id IN (SELECT t.release_id FROM release_texts t WHERE ${match.sql})`)
|
|
bindings.push(match.binding)
|
|
}
|
|
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
|
if (params.complete) where.push('r.complete')
|
|
// Incomplete only when final: a provisional tail release may yet get its missing files.
|
|
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
|
|
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
|
if (params.source) {
|
|
const code = (SOURCES as Record<string, number>)[params.source]
|
|
if (code === undefined) where.push('false')
|
|
else {
|
|
where.push('r.source = ?')
|
|
bindings.push(code)
|
|
}
|
|
}
|
|
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
|
|
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
|
|
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
|
|
let posters: number[] | null = null
|
|
if (params.poster) {
|
|
const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
|
posters = matched.map((m: { id: string }) => Number(m.id))
|
|
}
|
|
if (params.categories?.length) {
|
|
where.push('r.category = ANY(?)')
|
|
bindings.push(params.categories)
|
|
}
|
|
if (params.maxAgeDays) {
|
|
where.push('r.posted_at >= now() - make_interval(days => ?)')
|
|
bindings.push(params.maxAgeDays)
|
|
}
|
|
if (params.minSize !== undefined) {
|
|
where.push('r.bytes >= ?')
|
|
bindings.push(params.minSize)
|
|
}
|
|
if (params.maxSize !== undefined) {
|
|
where.push('r.bytes <= ?')
|
|
bindings.push(params.maxSize)
|
|
}
|
|
if (params.season !== undefined) {
|
|
const s = params.season
|
|
const e = params.episode
|
|
where.push('r.name ~* ?')
|
|
bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
|
|
}
|
|
|
|
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
|
const page = Math.max(params.page ?? 1, 1)
|
|
const offset = params.offset ?? (page - 1) * perPage
|
|
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
|
const whereSql = where.join(' AND ')
|
|
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
|
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
|
|
// Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage
|
|
// in the order asked; then merged.
|
|
const source =
|
|
posters === null
|
|
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
|
|
: {
|
|
from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
|
SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
|
where: 'true',
|
|
bindings: [posters] as unknown[],
|
|
}
|
|
const counting =
|
|
posters === null
|
|
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
|
|
: db.rawQuery(
|
|
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
|
SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
|
[posters, ...bindings]
|
|
)
|
|
const [{ rows }, { rows: counted }] = await Promise.all([
|
|
db.rawQuery(
|
|
`SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
|
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
|
|
r.settled_at IS NOT NULL AS final
|
|
FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id
|
|
WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
|
[...source.bindings, ...bindings, perPage, offset]
|
|
),
|
|
counting,
|
|
])
|
|
return {
|
|
page,
|
|
perPage,
|
|
offset,
|
|
total: Math.min(Number(counted[0].total), countLimit),
|
|
/** More than `total` match (the count stops at `countLimit`). */
|
|
totalCapped: Number(counted[0].total) > countLimit,
|
|
releases: (rows as any[]).map((r): ReleaseSummary => ({
|
|
id: Number(r.id),
|
|
name: r.name,
|
|
source: sourceName(r.source),
|
|
poster: r.poster ?? null,
|
|
groups: r.groups ?? [],
|
|
postedAt: r.posted_at,
|
|
fileTotal: r.file_total,
|
|
fileCount: Number(r.files),
|
|
size: Number(r.bytes),
|
|
readable: r.readable,
|
|
complete: r.complete,
|
|
final: r.final,
|
|
category: r.category,
|
|
settled: r.settled_at !== null,
|
|
passworded: r.passworded,
|
|
passwordKnown: r.password_known,
|
|
innerName: r.inner_name,
|
|
})),
|
|
}
|
|
}
|
|
|
|
/** A release with its files, read from its chunks. */
|
|
async release(id: number) {
|
|
const {
|
|
rows: [r],
|
|
} = await db.rawQuery(
|
|
`SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`,
|
|
[id]
|
|
)
|
|
if (!r) return null
|
|
const [files, { rows: inspections }, passwords] = await Promise.all([
|
|
NzbService.files(id),
|
|
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
|
NzbService.passwords(id),
|
|
])
|
|
return {
|
|
id: Number(r.id),
|
|
name: r.name,
|
|
source: sourceName(r.source),
|
|
poster: r.poster ?? null,
|
|
groups: r.groups ?? [],
|
|
postedAt: r.posted_at,
|
|
fileTotal: r.file_total,
|
|
fileCount: Number(r.files),
|
|
size: Number(r.bytes),
|
|
segments: Number(r.segments),
|
|
readable: r.readable,
|
|
complete: r.complete,
|
|
final: r.settled_at !== null,
|
|
hasPar2: r.has_par2,
|
|
category: r.category,
|
|
firstSeenAt: r.first_seen_at,
|
|
settledAt: r.settled_at,
|
|
passworded: r.passworded,
|
|
/** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
|
|
passwords,
|
|
innerName: r.inner_name,
|
|
inspectedAt: r.inspected_at,
|
|
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
|
|
inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
|
|
files: files.map((f) => ({
|
|
name: f.name,
|
|
subject: f.subject,
|
|
poster: f.poster,
|
|
postedAt: f.postedAt === null ? null : new Date(f.postedAt),
|
|
size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
|
|
parts: f.segments.length,
|
|
})),
|
|
}
|
|
}
|
|
|
|
/** Totals for the web interface, computed at most once a minute (callers in between share them). */
|
|
async stats() {
|
|
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
|
this.statsRefresh ??= Promise.race([
|
|
this.computeStats(),
|
|
new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
|
|
])
|
|
.then((value) => {
|
|
this.statsCache = { at: Date.now(), value }
|
|
})
|
|
.catch((error) => {
|
|
logger.warn(`Stats: ${error.message}`)
|
|
if (!this.statsCache) throw error
|
|
})
|
|
.finally(() => {
|
|
this.statsRefresh = undefined
|
|
})
|
|
if (!this.statsCache) await this.statsRefresh
|
|
}
|
|
return this.statsCache!.value
|
|
}
|
|
|
|
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
|
private statsRefresh?: Promise<void>
|
|
|
|
/**
|
|
* Estimates (a catalog of 100M+ rows can't be counted on every visit):
|
|
* the row count from Postgres's statistics, below a million rows exact;
|
|
* readable share, files and size from a sample of 5,000 rows; ranges
|
|
* counted exactly (a few hundred thousand rows).
|
|
*/
|
|
private async computeStats() {
|
|
const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
|
|
const estimate = Number(rows[0]?.n ?? 0)
|
|
// A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
|
|
const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
|
|
const { rows: s } = await db.rawQuery(
|
|
`SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
|
|
FROM releases ${sample} WHERE files > 0`
|
|
)
|
|
const sampled = Number(s[0].sampled)
|
|
const releases = sample ? Math.max(estimate, 0) : sampled
|
|
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
|
// History by articles; the pass: the history ranges before those deferred behind them (priority
|
|
// 2000 apart; e.g. groups added later that go first).
|
|
const { rows: ranges } = await db.rawQuery(
|
|
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0 AND status <> 'ancient'),
|
|
pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
|
|
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges WHERE status <> 'ancient') AS total,
|
|
coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
|
|
coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
|
|
coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
|
|
(SELECT max(done_at) FROM ranges) AS last_done_at,
|
|
(SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
|
|
FROM h`
|
|
)
|
|
const g = ranges[0]
|
|
return {
|
|
approximate: sample !== '',
|
|
releases,
|
|
readableReleases: scale(s[0].readable),
|
|
files: scale(s[0].files),
|
|
size: scale(s[0].bytes),
|
|
ranges: { done: Number(g.done), total: Number(g.total) },
|
|
/** History and the current pass, in articles. */
|
|
history: { done: Number(g.history_done), total: Number(g.history) },
|
|
pass: { done: Number(g.pass_done), total: Number(g.pass) },
|
|
lastIndexedAt: g.last_done_at,
|
|
tailIndexedAt: g.tail_done_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
export default new SearchService()
|