diff --git a/.adonisjs/server/controllers.ts b/.adonisjs/server/controllers.ts index 3c9dd4e..844c17e 100644 --- a/.adonisjs/server/controllers.ts +++ b/.adonisjs/server/controllers.ts @@ -6,6 +6,7 @@ export const controllers = { AccessTokens: () => import('#controllers/access_tokens_controller'), NewAccount: () => import('#controllers/new_account_controller'), + Newznab: () => import('#controllers/newznab_controller'), Node: () => import('#controllers/node_controller'), Profile: () => import('#controllers/profile_controller'), Releases: () => import('#controllers/releases_controller'), diff --git a/.gitignore b/.gitignore index dd7579a..8dba94d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ yarn-error.log # Platform specific .DS_Store + +# Generated by the dev tooling +.adonisjs/server/routes.json diff --git a/adonisrc.ts b/adonisrc.ts index 71a369f..77c1c33 100644 --- a/adonisrc.ts +++ b/adonisrc.ts @@ -53,6 +53,7 @@ export default defineConfig({ () => import('@adonisjs/cors/cors_provider'), () => import('@adonisjs/auth/auth_provider'), () => import('#providers/api_provider'), + () => import('@adonisjs/static/static_provider') ], /* @@ -103,7 +104,10 @@ export default defineConfig({ | the production build. | */ - metaFiles: [], + metaFiles: [{ + pattern: 'public/**', + reloadServer: false, + }], hooks: { init: [ diff --git a/app/controllers/newznab_controller.ts b/app/controllers/newznab_controller.ts new file mode 100644 index 0000000..0f17e02 --- /dev/null +++ b/app/controllers/newznab_controller.ts @@ -0,0 +1,115 @@ +import type { HttpContext } from '@adonisjs/core/http' +import env from '#start/env' +import NzbService from '#services/nzb_service' +import SearchService from '#services/search_service' +import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index' + +const LIMITS = { max: 100, default: 50 } + +/** Search functions and the category each defaults to when `cat` is absent. */ +const SEARCH_FUNCTIONS: Record = { + 'search': undefined, + 'tvsearch': 5000, + 'tv-search': 5000, + 'movie': 2000, + 'movie-search': 2000, + 'music': 3000, + 'audio': 3000, + 'book': 7000, +} + +/** The configured key, if any. */ +function configuredApiKey(): string | undefined { + const value: any = env.get('NEWZNAB_API_KEY') + const key = typeof value === 'string' ? value : value?.release() + return key || undefined +} + +function int(value: unknown): number | undefined { + return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined +} + +/** + * Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1): + * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=… + * Releases with readable names that aren't known to miss files. + */ +export default class NewznabController { + async handle({ request, response }: HttpContext) { + const qs = request.qs() + const t = String(qs.t ?? '') + const server = this.server(request) + const xml = (body: string, type = 'application/xml') => { + response.header('Content-Type', `${type}; charset=utf-8`) + return response.send(body) + } + + const apiKey = configuredApiKey() + if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials')) + if (t === 'caps') return xml(capsXml(server, LIMITS)) + + if (t === 'get') { + const id = int(qs.id) + if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)')) + const nzb = await NzbService.build(id) + if (!nzb) return xml(errorXml(300, 'No such item')) + const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb` + response.header('Content-Disposition', `attachment; filename="${filename}"`) + return xml(nzb.xml, 'application/x-nzb') + } + + if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`)) + + const requested = String(qs.cat ?? '') + .split(',') + .map((c) => int(c.trim())) + .filter((c): c is number => c !== undefined) + const fallback = SEARCH_FUNCTIONS[t] + const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : []) + const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max) + const offset = int(qs.offset) ?? 0 + + // ID-only lookups (tvdbid, imdbid, …) aren't supported: an empty result rather than everything. + const q = typeof qs.q === 'string' ? qs.q.trim() : '' + const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p]) + const result = idOnly + ? null + : await SearchService.search({ + q, + categories, + readable: true, + notIncomplete: true, + maxAgeDays: int(qs.maxage), + season: int(qs.season), + episode: int(qs.season) !== undefined ? int(qs.ep) : undefined, + sort: 'newest', + perPage: limit, + offset, + }) + + const items = (result?.releases ?? []).map((r) => { + const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name) + return { + id: r.id, + title: r.name, + category: r.category ?? 8010, + size: r.size, + files: r.fileCount, + postedAt: r.postedAt ? new Date(r.postedAt) : null, + poster: r.poster, + groups: r.groups ?? [], + // Passwords come with phase 2 (archive inspection). + password: false, + season: tv ? `S${tv[1].padStart(2, '0')}` : undefined, + episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined, + } + }) + return xml(resultsXml(server, items, offset, result?.total ?? 0), 'application/rss+xml') + } + + private server(request: HttpContext['request']): NewznabServer { + const proto = request.header('x-forwarded-proto') ?? 'http' + const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost' + return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey: configuredApiKey() } + } +} diff --git a/app/controllers/releases_controller.ts b/app/controllers/releases_controller.ts index 1f6e1dd..b551e05 100644 --- a/app/controllers/releases_controller.ts +++ b/app/controllers/releases_controller.ts @@ -1,27 +1,60 @@ import type { HttpContext } from '@adonisjs/core/http' -import db from '@adonisjs/lucid/services/db' import NzbService from '#services/nzb_service' +import SearchService, { type SortOrder } from '#services/search_service' +import { expandCategories } from '#lib/newznab/index' -/** The catalog (REBUILD.md, part 9): a name search and a release's NZB. */ +const flag = (value: unknown) => value === '1' || value === 'true' || value === true + +/** A size in MB from the query string, as bytes. */ +function megabytes(value: unknown): number | undefined { + const mb = Number(value) + return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined +} + +/** The catalog for the web interface (REBUILD.md, part 9). */ export default class ReleasesController { - /** Releases whose name has every word of `q` (dots, dashes, underscores and spaces alike), newest first. */ + /** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */ async index({ request }: HttpContext) { - const q = String(request.input('q', '')).trim() - const limit = Math.min(Number(request.input('limit', 50)) || 50, 200) - const query = db - .from('releases') - .select('id', 'name', 'poster', 'groups', 'posted_at', 'files', 'file_total', 'bytes', 'complete', 'readable') - .orderBy('posted_at', 'desc') - .limit(limit) - for (const word of q.split(/[\s._-]+/).filter(Boolean)) query.whereILike('name', `%${word.replace(/[\\%_]/g, '\\$&')}%`) - return { releases: await query } + const qs = request.qs() + const categories = String(qs.cat ?? '') + .split(',') + .map((c) => Number(c.trim())) + .filter((c) => Number.isInteger(c) && c > 0) + return SearchService.search({ + q: typeof qs.q === 'string' ? qs.q : undefined, + categories: categories.length ? expandCategories(categories) : undefined, + readable: flag(qs.readable), + complete: flag(qs.complete), + source: typeof qs.source === 'string' && qs.source ? qs.source : undefined, + poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined, + minSize: megabytes(qs.min_size), + maxSize: megabytes(qs.max_size), + sort: qs.sort as SortOrder | undefined, + page: Number(qs.page) || 1, + perPage: Number(qs.per_page) || 50, + // The page shows "1,000+"; counting further made filter clicks slow. + countLimit: 1000, + }) } + /** GET /api/v1/releases/:id, with its files */ + async show({ params, response }: HttpContext) { + const release = await SearchService.release(Number(params.id)) + return release ?? response.notFound({ error: 'Release not found' }) + } + + /** GET /api/v1/releases/:id/nzb */ async nzb({ params, response }: HttpContext) { const nzb = await NzbService.build(Number(params.id)) - if (!nzb) return response.notFound({ error: 'No such release' }) + if (!nzb) return response.notFound({ error: 'Release not found' }) + const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb` response.header('Content-Type', 'application/x-nzb') - response.header('Content-Disposition', `attachment; filename="${nzb.name.replace(/["\\/]/g, '_')}.nzb"`) - return nzb.xml + response.header('Content-Disposition', `attachment; filename="${filename}"`) + return response.send(nzb.xml) + } + + /** GET /api/v1/stats */ + async stats() { + return SearchService.stats() } } diff --git a/app/services/nzb_service.ts b/app/services/nzb_service.ts index 931fd06..2f82dcb 100644 --- a/app/services/nzb_service.ts +++ b/app/services/nzb_service.ts @@ -1,31 +1,34 @@ import db from '@adonisjs/lucid/services/db' -import { decodeChunk } from '#lib/chunks/index' +import { decodeChunk, type ChunkFile } from '#lib/chunks/index' import { buildNzb } from '#lib/nzb/index' /** - * A release's NZB (REBUILD.md, part 9), from its chunks in the chunk store: - * every chunk decoded, its files in name order. A file stored twice (the same - * name and first article, e.g. a range read again) is taken once. + * A release's files and NZB (REBUILD.md, part 9), from its chunks in the + * chunk store: every chunk decoded, its files in name order. A file stored + * twice (the same name and first article, e.g. a range read again) is taken + * once. */ class NzbService { - async build(releaseId: number): Promise<{ name: string; xml: string } | null> { - const release = await db.from('releases').where('id', releaseId).select('name').first() - if (!release) return null + async files(releaseId: number): Promise { const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload') - const files = new Map[number]>() + const files = new Map() for (const chunk of chunks) { for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file) } + return [...files.values()].sort((a, b) => a.name.localeCompare(b.name)) + } + + async build(releaseId: number): Promise<{ name: string; xml: string } | null> { + const release = await db.from('releases').where('id', releaseId).select('name').first() + if (!release) return null const xml = buildNzb( - [...files.values()] - .sort((a, b) => a.name.localeCompare(b.name)) - .map((f) => ({ - name: f.name, - poster: f.poster, - date: f.postedAt ?? 0, - groups: f.groups, - segments: f.segments.map(([number, id, size]) => ({ number, id, size })), - })), + (await this.files(releaseId)).map((f) => ({ + name: f.name, + poster: f.poster, + date: f.postedAt ?? 0, + groups: f.groups, + segments: f.segments.map(([number, id, size]) => ({ number, id, size })), + })), { title: release.name } ) return { name: release.name, xml } diff --git a/app/services/search_service.ts b/app/services/search_service.ts new file mode 100644 index 0000000..348fd38 --- /dev/null +++ b/app/services/search_service.ts @@ -0,0 +1,246 @@ +import db from '@adonisjs/lucid/services/db' +import NzbService from '#services/nzb_service' + +export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name' + +/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */ +export interface SearchParams { + q?: string + /** Only releases with a file whose name is readable (not an obfuscated token). */ + readable?: boolean + /** Only releases known to have all their files. */ + complete?: boolean + /** Leave out releases known to miss files (Newznab). */ + notIncomplete?: boolean + /** How the release was recognised: stem, subject or post. */ + source?: string + poster?: string + /** Exact category ids (already expanded from parents). */ + categories?: number[] + maxAgeDays?: number + /** TV: season and episode, matched as S01E02 or 1x02 in the name. */ + season?: number + episode?: number + /** Size bounds in bytes. */ + minSize?: number + maxSize?: number + sort?: SortOrder + page?: number + perPage?: number + /** Overrides `page` (Newznab pages by offset). */ + offset?: number + /** Count matches up to this many past the page; `total` is capped there. */ + countLimit?: number +} + +/** A release in search results. */ +export interface ReleaseSummary { + id: number + name: string + source: string + poster: string | null + groups: string[] + postedAt: Date | null + fileTotal: number | null + fileCount: number + size: number + readable: boolean + complete: boolean | null + category: number + settled: boolean +} + +const SORTS: Record = { + newest: 'r.posted_at DESC NULLS LAST, r.id DESC', + oldest: 'r.posted_at ASC NULLS LAST, r.id ASC', + largest: 'r.bytes DESC, r.id DESC', + files: 'r.files DESC, r.id DESC', + name: 'r.name ASC, r.id ASC', +} + +/** A broad search needn't count every match (each counted row is a heap read). */ +const COUNT_LIMIT = 10_000 +const STATS_TTL_MS = 60 * 1000 + +/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */ +function contains(word: string): string { + return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%` +} + +/** Searching and browsing the catalog, for the web interface and Newznab. */ +class SearchService { + async search(params: SearchParams) { + const where: string[] = ['r.files > 0'] + const bindings: unknown[] = [] + // Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike. + for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) { + where.push('r.name ILIKE ?') + bindings.push(contains(word)) + } + if (params.readable) where.push('r.readable') + if (params.complete) where.push('r.complete') + if (params.notIncomplete) where.push('r.complete IS NOT FALSE') + if (params.source) { + where.push('r.source = ?') + bindings.push(params.source) + } + if (params.poster) { + where.push('r.poster ILIKE ?') + bindings.push(contains(params.poster)) + } + if (params.categories?.length) { + where.push('r.category = ANY(?)') + bindings.push(params.categories) + } + if (params.maxAgeDays) { + where.push('r.posted_at >= now() - make_interval(days => ?)') + bindings.push(params.maxAgeDays) + } + if (params.minSize !== undefined) { + where.push('r.bytes >= ?') + bindings.push(params.minSize) + } + if (params.maxSize !== undefined) { + where.push('r.bytes <= ?') + bindings.push(params.maxSize) + } + if (params.season !== undefined) { + const s = params.season + const e = params.episode + where.push('r.name ~* ?') + bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`) + } + + const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200) + const page = Math.max(params.page ?? 1, 1) + const offset = params.offset ?? (page - 1) * perPage + const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest + const whereSql = where.join(' AND ') + const countLimit = offset + (params.countLimit ?? COUNT_LIMIT) + const [{ rows }, { rows: counted }] = await Promise.all([ + db.rawQuery( + `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at + FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`, + [...bindings, perPage, offset] + ), + db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings), + ]) + return { + page, + perPage, + offset, + total: Math.min(Number(counted[0].total), countLimit), + /** More than `total` match (the count stops at `countLimit`). */ + totalCapped: Number(counted[0].total) > countLimit, + releases: (rows as any[]).map((r): ReleaseSummary => ({ + id: Number(r.id), + name: r.name, + source: r.source, + poster: r.poster, + groups: r.groups, + postedAt: r.posted_at, + fileTotal: r.file_total, + fileCount: Number(r.files), + size: Number(r.bytes), + readable: r.readable, + complete: r.complete, + category: r.category, + settled: r.settled_at !== null, + })), + } + } + + /** A release with its files, read from its chunks. */ + async release(id: number) { + const r = await db.from('releases').where('id', id).first() + if (!r) return null + const files = await NzbService.files(id) + return { + id: Number(r.id), + name: r.name, + source: r.source, + poster: r.poster, + groups: r.groups, + postedAt: r.posted_at, + fileTotal: r.file_total, + fileCount: Number(r.files), + size: Number(r.bytes), + segments: Number(r.segments), + readable: r.readable, + complete: r.complete, + hasPar2: r.has_par2, + category: r.category, + firstSeenAt: r.first_seen_at, + lastSeenAt: r.last_seen_at, + settledAt: r.settled_at, + files: files.map((f) => ({ + name: f.name, + subject: f.subject, + poster: f.poster, + postedAt: f.postedAt === null ? null : new Date(f.postedAt), + size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0), + parts: f.segments.length, + })), + } + } + + /** Totals for the web interface, computed at most once a minute (callers in between share them). */ + async stats() { + if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) { + this.statsRefresh ??= this.computeStats() + .then((value) => { + this.statsCache = { at: Date.now(), value } + }) + .catch((error) => { + if (!this.statsCache) throw error + }) + .finally(() => { + this.statsRefresh = undefined + }) + if (!this.statsCache) await this.statsRefresh + } + return this.statsCache!.value + } + + private statsCache?: { at: number; value: Awaited> } + private statsRefresh?: Promise + + /** + * Estimates (a catalog of 100M+ rows can't be counted on every visit): + * the row count from Postgres's statistics, below a million rows exact; + * readable share, files and size from a sample; ranges counted exactly + * (a few hundred thousand rows). + */ + private async computeStats() { + const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`) + const estimate = Number(rows[0]?.n ?? 0) + const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM (0.5)' + const { rows: s } = await db.rawQuery( + `SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes + FROM releases ${sample} WHERE files > 0` + ) + const sampled = Number(s[0].sampled) + const releases = sample ? Math.max(estimate, 0) : sampled + const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0) + const { rows: ranges } = await db.rawQuery( + `SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total, + count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history, + max(done_at) AS last_done_at, + max(done_at) FILTER (WHERE priority < 0) AS tail_done_at + FROM ranges` + ) + const g = ranges[0] + return { + approximate: sample !== '', + releases, + readableReleases: scale(s[0].readable), + files: scale(s[0].files), + size: scale(s[0].bytes), + ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) }, + lastIndexedAt: g.last_done_at, + tailIndexedAt: g.tail_done_at, + } + } +} + +export default new SearchService() diff --git a/config/static.ts b/config/static.ts new file mode 100644 index 0000000..8d2878c --- /dev/null +++ b/config/static.ts @@ -0,0 +1,17 @@ +import { defineConfig } from '@adonisjs/static' + +/** + * Configuration options to tweak the static files middleware. + * The complete set of options are documented on the + * official documentation website. + * + * https://docs.adonisjs.com/guides/static-assets + */ +const staticServerConfig = defineConfig({ + enabled: true, + etag: true, + lastModified: true, + dotFiles: 'ignore', +}) + +export default staticServerConfig \ No newline at end of file diff --git a/database/migrations/1770000001200_release_category.ts b/database/migrations/1770000001200_release_category.ts new file mode 100644 index 0000000..73002a9 --- /dev/null +++ b/database/migrations/1770000001200_release_category.ts @@ -0,0 +1,39 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * A release's Newznab category from its name (REBUILD.md, part 9): voyager + * 1's release_category() without the archive contents (phase 2 has them). + * TV (5030/5040/5045), movies (2030/2040/2045), audio (3010/3040), + * books (7020), otherwise 8010. + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(String.raw` + CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$ + SELECT CASE + -- TV: S01E02, 1x02, or a season pack (S01). + WHEN name ~* '(^|[._ -])(S\d{1,2}E\d{1,3}|\d{1,2}x\d{2})([._ -]|$)' OR name ~* '[._ -]S\d{1,2}([._ -]|$)' THEN + CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045 + WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END + -- Movies: a year plus a source or quality marker. + WHEN name ~* '(^|[._ (-])(19|20)\d{2}([._ )-]|$)' + AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.?26[45]|hevc|remux)' THEN + CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045 + WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END + -- Audio: lossless, mp3, or scene music names (-WEB-2026-GROUP). + WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040 + WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\d{2}-[A-Za-z0-9]+$' THEN 3010 + WHEN name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020 + ELSE 8010 + END + $$`) + this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS category integer GENERATED ALWAYS AS (release_category(name)) STORED`) + // Newznab and the web interface list a category newest first. + this.schema.raw(`CREATE INDEX IF NOT EXISTS releases_category_posted_idx ON releases (category, posted_at DESC)`) + } + + async down() { + this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS category`) + this.schema.raw(`DROP FUNCTION IF EXISTS release_category(text)`) + } +} diff --git a/lib/newznab/categories.ts b/lib/newznab/categories.ts new file mode 100644 index 0000000..9e0f17a --- /dev/null +++ b/lib/newznab/categories.ts @@ -0,0 +1,40 @@ +/** Newznab categories this indexer assigns (release_category() in SQL, migration …1200). */ +export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [ + { id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] }, + { id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] }, + { id: 4000, name: 'PC', subcats: [] }, + { id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] }, + { id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] }, + { id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] }, +] + +/** Parent category of a (sub)category, e.g. 5040 → 5000. */ +export function parentCategory(id: number): number { + return Math.floor(id / 1000) * 1000 +} + +/** "TV > HD" style label. */ +export function categoryLabel(id: number): string { + const parent = CATEGORIES.find((c) => c.id === parentCategory(id)) + if (!parent) return 'Other' + const sub = parent.subcats.find((s) => s.id === id) + return sub ? `${parent.name} > ${sub.name}` : parent.name +} + +/** + * Expands requested categories (`cat=5000,2040`) to the concrete ones to + * match: a parent includes all of its subcategories. + */ +export function expandCategories(requested: number[]): number[] { + const result = new Set() + for (const id of requested) { + const parent = CATEGORIES.find((c) => c.id === id) + if (parent) { + result.add(parent.id) + for (const sub of parent.subcats) result.add(sub.id) + } else { + result.add(id) + } + } + return [...result] +} diff --git a/lib/newznab/index.ts b/lib/newznab/index.ts new file mode 100644 index 0000000..e28069d --- /dev/null +++ b/lib/newznab/index.ts @@ -0,0 +1,5 @@ +/** + * Newznab API formatting, independent of AdonisJS. From voyager 1 (its CONCEPT.md D23). + */ +export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js' +export { capsXml, errorXml, resultsXml, type NewznabItem, type NewznabServer } from './xml.js' diff --git a/lib/newznab/xml.ts b/lib/newznab/xml.ts new file mode 100644 index 0000000..ff73d4a --- /dev/null +++ b/lib/newznab/xml.ts @@ -0,0 +1,107 @@ +import { create } from 'xmlbuilder2' +import { CATEGORIES, categoryLabel, parentCategory } from './categories.js' + +const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/' + +export interface NewznabItem { + id: number + title: string + category: number + size: number + files: number + postedAt: Date | null + poster: string | null + groups: string[] + password: boolean + /** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */ + description?: string + season?: string + episode?: string +} + +export interface NewznabServer { + title: string + /** Absolute base URL, e.g. http://homelab:3333 */ + baseUrl: string + /** Appended to download links so a client can fetch without extra config. */ + apiKey?: string +} + +/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */ +/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */ +export function ubbToText(ubb: string): string { + return ubb + .replace(/|\[br\]/gi, '\n') + .replace(/<\/?[a-z][^>]*>/gi, '') + .replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim())) + .replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '') + .trim() +} + +export function errorXml(code: number, description: string): string { + return create({ version: '1.0', encoding: 'UTF-8' }) + .ele('error', { code: String(code), description }) + .end({ prettyPrint: true }) +} + +export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string { + const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps') + caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` }) + caps.ele('limits', { max: String(limits.max), default: String(limits.default) }) + caps.ele('registration', { available: 'no', open: 'no' }) + const searching = caps.ele('searching') + searching.ele('search', { available: 'yes', supportedParams: 'q' }) + searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' }) + searching.ele('movie-search', { available: 'yes', supportedParams: 'q' }) + searching.ele('audio-search', { available: 'yes', supportedParams: 'q' }) + searching.ele('book-search', { available: 'yes', supportedParams: 'q' }) + const categories = caps.ele('categories') + for (const category of CATEGORIES) { + const element = categories.ele('category', { id: String(category.id), name: category.name }) + for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name }) + } + return caps.end({ prettyPrint: true }) +} + +export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string { + const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', { + 'version': '2.0', + 'xmlns:atom': 'http://www.w3.org/2005/Atom', + 'xmlns:newznab': NEWZNAB_NS, + }) + const channel = rss.ele('channel') + channel.ele('title').txt(server.title) + channel.ele('description').txt(`${server.title} API results`) + channel.ele('link').txt(`${server.baseUrl}/`) + channel.ele('newznab:response', { offset: String(offset), total: String(total) }) + + const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : '' + for (const item of items) { + const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}` + const pubDate = (item.postedAt ?? new Date()).toUTCString() + const element = channel.ele('item') + element.ele('title').txt(item.title) + element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id)) + element.ele('link').txt(getUrl) + element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`) + element.ele('pubDate').txt(pubDate) + element.ele('category').txt(categoryLabel(item.category)) + element.ele('description').txt(item.description ? ubbToText(item.description) : item.title) + element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' }) + + const attr = (name: string, value: string | number) => + element.ele('newznab:attr', { name, value: String(value) }) + attr('category', parentCategory(item.category)) + if (item.category !== parentCategory(item.category)) attr('category', item.category) + attr('size', item.size) + attr('files', item.files) + attr('grabs', 0) + attr('password', item.password ? 1 : 0) + if (item.poster) attr('poster', item.poster) + if (item.groups.length) attr('group', item.groups[0]) + attr('usenetdate', pubDate) + if (item.season) attr('season', item.season) + if (item.episode) attr('episode', item.episode) + } + return rss.end({ prettyPrint: true }) +} diff --git a/package-lock.json b/package-lock.json index 33b91dd..5dd5fbd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@adonisjs/lucid": "^22.4.2", "@adonisjs/session": "^8.1.0", "@adonisjs/shield": "^9.0.0", + "@adonisjs/static": "^2.0.1", "@japa/api-client": "^3.2.1", "@tuyau/core": "^1.2.2", "@vinejs/vine": "^4.4.0", @@ -700,6 +701,27 @@ } } }, + "node_modules/@adonisjs/static": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@adonisjs/static/-/static-2.0.1.tgz", + "integrity": "sha512-pXSyQgha1yfyQDXPp0ywv7hJqk4JgCCkzWLkku2PW3Y6i/9GUTyzzNTCpz29loZzSF0b/EpiHXQE/sqBhW+p7w==", + "license": "MIT", + "dependencies": { + "serve-static": "^2.2.1" + }, + "engines": { + "node": ">=24.0.0" + }, + "peerDependencies": { + "@adonisjs/assembler": "^8.0.0-next.7 || ^8.0.0", + "@adonisjs/core": "^7.0.0-next.0 || ^7.0.0" + }, + "peerDependenciesMeta": { + "@adonisjs/assembler": { + "optional": true + } + } + }, "node_modules/@adonisjs/tsconfig": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@adonisjs/tsconfig/-/tsconfig-2.0.0.tgz", @@ -790,9 +812,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -809,9 +828,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -828,9 +844,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -847,9 +860,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1882,9 +1892,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -1902,9 +1909,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -1922,9 +1926,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -1942,9 +1943,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -1962,9 +1960,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -1982,9 +1977,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -3543,6 +3535,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -5548,6 +5546,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", @@ -6078,6 +6085,19 @@ "node": ">= 0.8" } }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/raw-body": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", @@ -6375,6 +6395,60 @@ "node": ">=10" } }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/send/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/set-cookie-parser": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", diff --git a/package.json b/package.json index fd5ee95..68da3d1 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "@adonisjs/lucid": "^22.4.2", "@adonisjs/session": "^8.1.0", "@adonisjs/shield": "^9.0.0", + "@adonisjs/static": "^2.0.1", "@japa/api-client": "^3.2.1", "@tuyau/core": "^1.2.2", "@vinejs/vine": "^4.4.0", diff --git a/start/env.ts b/start/env.ts index d953d64..165d92d 100644 --- a/start/env.ts +++ b/start/env.ts @@ -45,6 +45,9 @@ export default await Env.create(new URL('../', import.meta.url), { NNTP_CONNECTIONS: Env.schema.number.optional(), NNTP_TIMEOUT_MS: Env.schema.number.optional(), + // Newznab API key (empty: no key needed) + NEWZNAB_API_KEY: Env.schema.secret.optional(), + // Coordinator RANGE_ARTICLES: Env.schema.number.optional(), TAIL_EVERY_SECONDS: Env.schema.number.optional(), diff --git a/start/kernel.ts b/start/kernel.ts index d3c18be..5514e48 100644 --- a/start/kernel.ts +++ b/start/kernel.ts @@ -26,6 +26,7 @@ server.use([ () => import('#middleware/force_json_response_middleware'), () => import('#middleware/container_bindings_middleware'), () => import('@adonisjs/cors/cors_middleware'), + () => import('@adonisjs/static/static_middleware') ]) /** diff --git a/start/routes.ts b/start/routes.ts index f0ad8a8..7183fb2 100644 --- a/start/routes.ts +++ b/start/routes.ts @@ -11,9 +11,10 @@ import { middleware } from '#start/kernel' import router from '@adonisjs/core/services/router' import { controllers } from '#generated/controllers' -router.get('/', () => { - return { hello: 'world' } -}) +// The web interface is public/index.html (static middleware). +router.get('/health', () => ({ ok: true })) +// Newznab API for Prowlarr, Sonarr, Radarr (from voyager 1). +router.get('/api', [controllers.Newznab, 'handle']) router .group(() => { @@ -34,13 +35,11 @@ router .as('profile') .use(middleware.auth()) - // The catalog (REBUILD.md, part 9). - router - .group(() => { - router.get('releases', [controllers.Releases, 'index']) - router.get('releases/:id/nzb', [controllers.Releases, 'nzb']) - }) - .use(middleware.auth({ guards: ['api'] })) + // The catalog (REBUILD.md, part 9): open, as voyager 1's (the web interface uses it); Newznab has its key. + router.get('releases', [controllers.Releases, 'index']) + router.get('releases/:id', [controllers.Releases, 'show']).where('id', router.matchers.number()) + router.get('releases/:id/nzb', [controllers.Releases, 'nzb']).where('id', router.matchers.number()) + router.get('stats', [controllers.Releases, 'stats']) // Edge workers (REBUILD.md, parts 1–2). router diff --git a/tests/functional/releases_api.spec.ts b/tests/functional/releases_api.spec.ts index 8293dd5..b5a84e4 100644 --- a/tests/functional/releases_api.spec.ts +++ b/tests/functional/releases_api.spec.ts @@ -1,7 +1,6 @@ import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' -import User from '#models/user' import BatchStore from '#services/batch_store' import { RangeIndexer } from '#lib/edge/index' import { parseNzb } from '#lib/nzb/index' @@ -29,15 +28,13 @@ test.group('Releases API', (group) => { indexer.finish() await BatchStore.store(Number(r.id), Number(n.id), indexer.take()) - const user = await User.create({ email: 'u@example.invalid', password: 'secret-password' }) - const token = (await User.accessTokens.create(user)).value!.release() - ;(await client.get('/api/v1/releases?q=show s01e01')).assertStatus(401) - const found = await client.get('/api/v1/releases?q=show s01e01').bearerToken(token) + // Open, as voyager 1's catalog (the web interface uses it). + const found = await client.get('/api/v1/releases?q=show s01e01') found.assertStatus(200) const releases = (found.body() as { releases: { id: number; name: string }[] }).releases assert.lengthOf(releases, 1) - const nzb = await client.get(`/api/v1/releases/${releases[0].id}/nzb`).bearerToken(token) + const nzb = await client.get(`/api/v1/releases/${releases[0].id}/nzb`) nzb.assertStatus(200) assert.include(nzb.header('content-type'), 'application/x-nzb') const parsed = parseNzb(nzb.text())! @@ -45,4 +42,38 @@ test.group('Releases API', (group) => { assert.deepEqual(parsed.files.map((f) => f.segments.length), [1, 3]) assert.equal(parsed.files[1].segments[2].id, 'Some.Show.S01E01.1080p.WEB.part1.rar-3@test') }) + test('details with files, categories, stats and the Newznab API', async ({ client, assert }) => { + const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id') + const [n] = await db.table('nodes').insert({ name: 'api-node' }).returning('id') + const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: n.id }).returning('id') + const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } }) + indexer.add([...file('Some.Show.S01E01.1080p.WEB.mkv', 'Some.Show.S01E01.1080p.WEB [1/1]', 2, 100), ...file('Other.Movie.2024.2160p.BluRay.x265.mkv', 'Other.Movie.2024.2160p.BluRay.x265 [1/1]', 1, 300)], 1001) + indexer.finish() + await BatchStore.store(Number(r.id), Number(n.id), indexer.take()) + + // Web API: a category (TV, expanded to its subcategories) and a release's files. + const tv = (await client.get('/api/v1/releases?cat=5000')).body() as { releases: { id: number; name: string; category: number }[] } + assert.deepEqual(tv.releases.map((x) => [x.name, x.category]), [['Some Show S01E01 1080p WEB', 5040]]) + const detail = (await client.get(`/api/v1/releases/${tv.releases[0].id}`)).body() as { files: { name: string; parts: number; size: number }[] } + assert.deepEqual(detail.files, [{ ...detail.files[0], name: 'Some.Show.S01E01.1080p.WEB.mkv', parts: 2, size: 1400001 }]) + ;(await client.get('/api/v1/releases/999999')).assertStatus(404) + const stats = (await client.get('/api/v1/stats')).body() as { releases: number; ranges: { total: number } } + assert.equal(stats.releases, 2) + assert.equal(stats.ranges.total, 1) + + // Newznab: caps, a TV search with season and episode, the NZB; the key when one is set. + const caps = await client.get('/api?t=caps') + assert.include(caps.text(), '') + const search = await client.get('/api?t=tvsearch&q=some show&season=1&ep=1') + assert.include(search.text(), 'Some Show S01E01 1080p WEB') + assert.include(search.text(), '') + assert.notInclude(search.text(), 'Other Movie') + const movie = await client.get('/api?t=movie&cat=2045') + assert.include(movie.text(), 'Other Movie 2024 2160p BluRay x265') + const get = await client.get(`/api?t=get&id=${tv.releases[0].id}`) + assert.include(get.header('content-type'), 'application/x-nzb') + assert.equal(parseNzb(get.text())!.files.length, 1) + assert.include((await client.get('/api?t=get&id=999999')).text(), 'code="300"') + assert.include((await client.get('/api?t=nope')).text(), 'code="202"') + }) })