Daan MeijerandClaude Opus 5.5
fad1716d3a
Every probe rewrote its file's row on the HDD (content_type, archive): a new row version on a random page and an entry in each of the five indexes. They now live in file_details on the NVMe; files rows are inserted once. files:move-details copies the existing values after the deploy; a later migration drops the old columns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
422 lines
18 KiB
TypeScript
422 lines
18 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import { buildNzb } from '#lib/nzb/index'
|
|
import SpotService from '#services/SpotService'
|
|
import { segmentsOf } from '#services/FileStore'
|
|
import { spotCategoryLabels } from '#lib/spotnet/index'
|
|
|
|
/**
|
|
* `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
|
|
* without asking); `missing`: the archive is encrypted and we don't have
|
|
* it; `none`: no password needed as far as we know.
|
|
*/
|
|
export type PasswordState = 'known' | 'missing' | 'none'
|
|
|
|
export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
|
|
if (hasPassword) return 'known'
|
|
return encrypted ? 'missing' : 'none'
|
|
}
|
|
|
|
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
|
|
|
export interface SearchParams {
|
|
q?: string
|
|
/** Only releases whose name is readable (not an obfuscated token). */
|
|
readable?: boolean
|
|
/** Only releases that have all their files. */
|
|
complete?: boolean
|
|
/** Hide releases that need a password we don't have. */
|
|
hideEncrypted?: boolean
|
|
source?: string
|
|
poster?: string
|
|
sort?: SortOrder
|
|
page?: number
|
|
perPage?: number
|
|
/** Newznab: exact category ids (already expanded from parents). */
|
|
categories?: number[]
|
|
/** Newznab: leave out releases known to be missing files. */
|
|
notIncomplete?: boolean
|
|
maxAgeDays?: number
|
|
/** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
|
|
season?: number
|
|
episode?: number
|
|
/** Overrides `page` (Newznab uses offsets). */
|
|
offset?: number
|
|
/** Leave out releases that are only an encrypted container (nzbe). */
|
|
hideContainers?: boolean
|
|
/** Include incomplete releases that consist of par2 files only (hidden by default). */
|
|
includePar2Only?: boolean
|
|
/** Size bounds in bytes (releases without a known size never match). */
|
|
minSize?: number
|
|
maxSize?: number
|
|
/** File extensions (without the dot); a release matches if a file or archive entry has one of them (`releases.extensions`). */
|
|
extensions?: string[]
|
|
/** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
|
|
countLimit?: number
|
|
}
|
|
|
|
const SORTS: Record<SortOrder, string> = {
|
|
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
|
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
|
largest: 'r.size DESC, r.id DESC',
|
|
files: 'r.file_count DESC, r.id DESC',
|
|
name: 'r.name ASC, r.id ASC',
|
|
}
|
|
|
|
/**
|
|
* Shown in search: releases with files, and Spotnet listings whose NZB is
|
|
* fetched on first download (CONCEPT.md D29). Matches the partial indexes.
|
|
*/
|
|
const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
|
|
|
|
/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
|
|
const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
|
|
SELECT 1 FROM files f LEFT JOIN file_details d ON d.file_id = f.id WHERE f.release_id = r.id
|
|
AND d.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
|
|
|
|
/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
|
|
function pattern(word: string): string {
|
|
const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
|
|
return `%${escaped.replace(/[._\- ]+/g, '_')}%`
|
|
}
|
|
|
|
/**
|
|
* As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
|
|
* spaces as one space. `The.Block.AU` and `the block au` search the same.
|
|
*/
|
|
export function searchNormalize(text: string): string {
|
|
return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
|
|
}
|
|
|
|
/** How long the web UI's totals are reused. */
|
|
const STATS_TTL_MS = 60 * 1000
|
|
|
|
/**
|
|
* Counting stops here: a broad search needn't count every match. Each
|
|
* counted row is a random heap read, so the web UI counts less (it only
|
|
* shows "1,000+"); Newznab clients page by this total and keep the default.
|
|
*/
|
|
const COUNT_LIMIT = 10_000
|
|
|
|
/**
|
|
* Searching and browsing releases for the web interface (CONCEPT.md D22).
|
|
*/
|
|
class SearchService {
|
|
async search(params: SearchParams) {
|
|
const where: string[] = [
|
|
VISIBLE_SQL,
|
|
// Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
|
|
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
|
|
]
|
|
const bindings: unknown[] = []
|
|
|
|
// Every word must occur in the release's search text: its name, archive
|
|
// listing and file names (CONCEPT.md D33), one trigram index for all.
|
|
const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
|
|
for (const word of words) {
|
|
where.push(`r.search_text LIKE ?`)
|
|
bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
|
|
}
|
|
if (params.readable) where.push('is_readable_name(r.name)')
|
|
if (params.minSize !== undefined) {
|
|
where.push('r.size >= ?')
|
|
bindings.push(params.minSize)
|
|
}
|
|
if (params.maxSize !== undefined) {
|
|
where.push('r.size <= ?')
|
|
bindings.push(params.maxSize)
|
|
}
|
|
// File types from file names and archive listings (migration …130), via their GIN index.
|
|
const extensions = (params.extensions ?? []).map((e) => e.toLowerCase().replace(/^\./, '')).filter((e) => /^[a-z0-9]{1,8}$/.test(e)).slice(0, 8)
|
|
if (extensions.length > 0) {
|
|
where.push('r.extensions && ?::text[]')
|
|
bindings.push(extensions)
|
|
}
|
|
if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
|
|
if (params.hideContainers) where.push('r.container IS NULL')
|
|
// par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
|
|
if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
|
|
// Kept current by triggers (migration …860).
|
|
if (params.complete) where.push('r.complete')
|
|
if (params.source) {
|
|
where.push('r.source = ?')
|
|
bindings.push(params.source)
|
|
}
|
|
if (params.poster) {
|
|
where.push('r.poster ILIKE ?')
|
|
bindings.push(pattern(params.poster))
|
|
}
|
|
if (params.categories?.length) {
|
|
where.push('r.category = ANY(?)')
|
|
bindings.push(params.categories)
|
|
}
|
|
if (params.notIncomplete) {
|
|
where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
|
|
}
|
|
if (params.maxAgeDays) {
|
|
where.push(`r.posted_at >= now() - make_interval(days => ?)`)
|
|
bindings.push(params.maxAgeDays)
|
|
}
|
|
if (params.season !== undefined) {
|
|
const s = params.season
|
|
const e = params.episode
|
|
where.push('r.name ~* ?')
|
|
bindings.push(
|
|
e !== undefined
|
|
? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
|
|
: `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
|
|
)
|
|
}
|
|
|
|
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
|
const page = Math.max(params.page ?? 1, 1)
|
|
const offset = params.offset ?? (page - 1) * perPage
|
|
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
|
const whereSql = where.join(' AND ')
|
|
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
|
|
|
const [{ rows }, { rows: countRows }] = await Promise.all([
|
|
db.rawQuery(
|
|
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
|
|
r.size, r.encrypted, is_readable_name(r.name) AS readable,
|
|
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
|
|
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
|
|
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
|
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
|
|
jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
|
|
r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
|
|
(SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
|
|
FROM releases r WHERE ${whereSql}
|
|
ORDER BY ${order} LIMIT ? OFFSET ?`,
|
|
[...bindings, perPage, offset]
|
|
),
|
|
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
|
|
])
|
|
|
|
return {
|
|
page,
|
|
perPage,
|
|
offset,
|
|
total: Math.min(Number(countRows[0].total), countLimit),
|
|
/** More than `total` match (the count stops at `countLimit`). */
|
|
totalCapped: Number(countRows[0].total) > countLimit,
|
|
releases: rows.map((r: any) => ({
|
|
id: Number(r.id),
|
|
name: r.name,
|
|
source: r.source,
|
|
poster: r.poster,
|
|
groups: r.groups,
|
|
postedAt: r.posted_at,
|
|
fileTotal: r.file_total,
|
|
fileCount: r.file_count,
|
|
size: Number(r.size),
|
|
encrypted: r.encrypted,
|
|
readable: r.readable,
|
|
listedFiles: Number(r.listed_files),
|
|
listedPresent: Number(r.listed_present),
|
|
archiveEntries: Number(r.archive_entries),
|
|
complete: r.complete,
|
|
category: r.category,
|
|
container: r.container,
|
|
par2Only: r.par2_only,
|
|
listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
|
|
hasPassword: r.has_password,
|
|
passwordState: passwordState(r.encrypted, r.has_password),
|
|
spotDescription: r.spot_description,
|
|
})),
|
|
}
|
|
}
|
|
|
|
async release(id: number) {
|
|
const release = await db.from('releases').where('id', id).first()
|
|
if (!release) return null
|
|
|
|
const [files, listed, pending] = await Promise.all([
|
|
db
|
|
.from('files as f')
|
|
.leftJoin('binaries as b', 'b.file_id', 'f.id')
|
|
.where('f.release_id', id)
|
|
.orderByRaw('coalesce(f.real_name, f.filename)')
|
|
.select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
|
|
db.rawQuery(
|
|
`SELECT rf.name, rf.size,
|
|
EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
|
WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
|
|
FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
|
|
[id]
|
|
),
|
|
db.rawQuery(
|
|
`SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
|
|
coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
|
|
FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
|
|
[id]
|
|
),
|
|
])
|
|
|
|
const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
|
|
const spot = spotRow ? await SpotService.withDetails(spotRow) : null
|
|
return {
|
|
id: Number(release.id),
|
|
name: release.name,
|
|
source: release.source,
|
|
spot: spot
|
|
? {
|
|
messageId: spot.message_id,
|
|
title: spot.title,
|
|
tag: spot.tag,
|
|
poster: spot.poster,
|
|
postedAt: spot.posted_at,
|
|
website: spot.website,
|
|
description: spot.description,
|
|
image: spot.image?.segments
|
|
? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
|
|
: spot.image?.url
|
|
? { external: spot.image.url }
|
|
: null,
|
|
categories: spotCategoryLabels(spot.category, spot.subcats),
|
|
}
|
|
: null,
|
|
password: release.password ?? null,
|
|
passwordSource: release.password_source ?? null,
|
|
passwordState: passwordState(release.encrypted, release.password !== null),
|
|
listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
|
|
poster: release.poster,
|
|
groups: release.groups,
|
|
postedAt: release.posted_at,
|
|
fileTotal: release.file_total,
|
|
fileCount: release.file_count,
|
|
size: Number(release.size),
|
|
encrypted: release.encrypted,
|
|
container: release.container,
|
|
contents: release.contents ?? [],
|
|
files: files.map((f) => ({
|
|
id: Number(f.id),
|
|
name: f.real_name ?? f.filename,
|
|
postedName: f.filename,
|
|
realName: f.real_name,
|
|
size: Number(f.exact_size ?? f.size),
|
|
parts: f.parts,
|
|
postedAt: f.date ? new Date(Number(f.date)) : null,
|
|
poster: f.poster,
|
|
})),
|
|
par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
|
|
pending: {
|
|
binaries: Number(pending.rows[0].binaries),
|
|
partsHave: Number(pending.rows[0].parts_have),
|
|
partsTotal: Number(pending.rows[0].parts_total),
|
|
},
|
|
}
|
|
}
|
|
|
|
/** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
|
|
async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
|
|
const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
|
|
if (!spot) return null
|
|
const data = await SpotService.image(spot)
|
|
if (!data) return null
|
|
const type =
|
|
data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
|
|
: data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
|
|
: data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
|
|
: data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
|
|
: data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
|
|
: null
|
|
return type ? { data, type } : null
|
|
}
|
|
|
|
async nzb(id: number): Promise<{ name: string; xml: string } | null> {
|
|
let release = await db.from('releases').where('id', id).first()
|
|
if (!release) return null
|
|
if (release.source === 'spot' && Number(release.file_count) === 0) {
|
|
// A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
|
|
const imported = await SpotService.importListing(id)
|
|
if (!imported) return null
|
|
id = imported
|
|
release = await db.from('releases').where('id', id).first()
|
|
}
|
|
const files = await db
|
|
.from('files')
|
|
.where('release_id', id)
|
|
.orderByRaw('coalesce(real_name, filename)')
|
|
.select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
|
|
const xml = buildNzb(
|
|
files.map((f) => ({
|
|
name: f.real_name ?? f.filename,
|
|
poster: f.poster,
|
|
date: Number(f.date),
|
|
groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
|
|
segments: segmentsOf(f),
|
|
})),
|
|
{ title: release.name, password: release.password ?? undefined }
|
|
)
|
|
return { name: release.name, xml }
|
|
}
|
|
|
|
/**
|
|
* Totals for the web UI header. They count millions of rows (minutes with
|
|
* the backfill running), so they are computed at most once per
|
|
* STATS_TTL_MS and never twice at a time; callers in between share it.
|
|
*/
|
|
async stats() {
|
|
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
|
this.statsRefresh ??= this.computeStats()
|
|
.then((value) => {
|
|
this.statsCache = { at: Date.now(), value }
|
|
})
|
|
// A failed background refresh keeps the previous totals; only a caller that waits sees the error.
|
|
.catch((error) => {
|
|
if (!this.statsCache) throw error
|
|
})
|
|
.finally(() => {
|
|
this.statsRefresh = undefined
|
|
})
|
|
// Only the very first caller waits; later ones get the previous totals meanwhile.
|
|
if (!this.statsCache) await this.statsRefresh
|
|
}
|
|
return this.statsCache!.value
|
|
}
|
|
|
|
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
|
private statsRefresh?: Promise<void>
|
|
|
|
/**
|
|
* Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
|
|
* from Postgres's statistics (visible releases = the entries of an index
|
|
* that covers exactly them), ratios and sizes from a 0.5% sample of the
|
|
* releases table. Within a few percent once autovacuum has analysed.
|
|
*/
|
|
private async computeStats() {
|
|
const { rows } = await db.rawQuery(`
|
|
WITH sample AS (
|
|
SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
|
|
FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
|
|
SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
|
|
-- files is partitioned (D39): its partitions hold the counts.
|
|
(SELECT sum(c.reltuples) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass) AS files,
|
|
(SELECT count(*) FROM sample) AS sampled,
|
|
(SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
|
|
-- Indexed data only: a spot listing's size is what the spot announces.
|
|
(SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
|
|
(SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
|
|
(SELECT max(updated_at) FROM groups) AS last_indexed_at,
|
|
(SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
|
|
FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
|
|
const r = rows[0]
|
|
const releases = Math.max(0, Number(r.releases))
|
|
const sampled = Number(r.sampled)
|
|
// Sample counts scaled to all visible releases.
|
|
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
|
return {
|
|
approximate: true,
|
|
releases,
|
|
readableReleases: scale(r.sampled_readable),
|
|
files: Math.max(0, Number(r.files)),
|
|
size: scale(r.sampled_size),
|
|
encryptedReleases: scale(r.sampled_encrypted),
|
|
lastIndexedAt: r.last_indexed_at,
|
|
sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
|
|
}
|
|
}
|
|
}
|
|
|
|
export default new SearchService()
|