Rebuild: the catalog API and Newznab, from voyager 1

- /api (Newznab: caps, search, tvsearch, movie, music, book, get) for
  Prowlarr, Sonarr and Radarr: voyager 1's controller and XML on the new
  catalog; readable releases not known to miss files; NEWZNAB_API_KEY.
- /api/v1/releases (q, cat, readable, complete, source, poster, size,
  sort, paging), /api/v1/releases/:id with its files (decoded from the
  chunks; NzbService.files, shared with the NZB), /api/v1/stats
  (estimates, plus history and tail progress), /health. Open, as voyager
  1's catalog; the web interface uses it.
- release_category(name): voyager 1's categorizer without the archive
  contents, a generated column with a (category, posted_at) index.
- @adonisjs/static for the web interface (next).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 10:45:47 +02:00
1 parent 57dfa3d749
commit 8e52bff213
18 files changed
+801 -79

No files matched your search

+1
View File
@@ -6,6 +6,7 @@
export const controllers = {
AccessTokens: () => import('#controllers/access_tokens_controller'),
NewAccount: () => import('#controllers/new_account_controller'),
Newznab: () => import('#controllers/newznab_controller'),
Node: () => import('#controllers/node_controller'),
Profile: () => import('#controllers/profile_controller'),
Releases: () => import('#controllers/releases_controller'),
+3
View File
@@ -24,3 +24,6 @@ yarn-error.log
# Platform specific
.DS_Store
# Generated by the dev tooling
.adonisjs/server/routes.json
+5 -1
View File
@@ -53,6 +53,7 @@ export default defineConfig({
() => import('@adonisjs/cors/cors_provider'),
() => import('@adonisjs/auth/auth_provider'),
() => import('#providers/api_provider'),
() => import('@adonisjs/static/static_provider')
],
/*
@@ -103,7 +104,10 @@ export default defineConfig({
| the production build.
|
*/
metaFiles: [],
metaFiles: [{
pattern: 'public/**',
reloadServer: false,
}],
hooks: {
init: [
+115
View File
@@ -0,0 +1,115 @@
import type { HttpContext } from '@adonisjs/core/http'
import env from '#start/env'
import NzbService from '#services/nzb_service'
import SearchService from '#services/search_service'
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
const LIMITS = { max: 100, default: 50 }
/** Search functions and the category each defaults to when `cat` is absent. */
const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
'search': undefined,
'tvsearch': 5000,
'tv-search': 5000,
'movie': 2000,
'movie-search': 2000,
'music': 3000,
'audio': 3000,
'book': 7000,
}
/** The configured key, if any. */
function configuredApiKey(): string | undefined {
const value: any = env.get('NEWZNAB_API_KEY')
const key = typeof value === 'string' ? value : value?.release()
return key || undefined
}
function int(value: unknown): number | undefined {
return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined
}
/**
* Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1):
* GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
* Releases with readable names that aren't known to miss files.
*/
export default class NewznabController {
async handle({ request, response }: HttpContext) {
const qs = request.qs()
const t = String(qs.t ?? '')
const server = this.server(request)
const xml = (body: string, type = 'application/xml') => {
response.header('Content-Type', `${type}; charset=utf-8`)
return response.send(body)
}
const apiKey = configuredApiKey()
if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
if (t === 'caps') return xml(capsXml(server, LIMITS))
if (t === 'get') {
const id = int(qs.id)
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
const nzb = await NzbService.build(id)
if (!nzb) return xml(errorXml(300, 'No such item'))
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
response.header('Content-Disposition', `attachment; filename="${filename}"`)
return xml(nzb.xml, 'application/x-nzb')
}
if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
const requested = String(qs.cat ?? '')
.split(',')
.map((c) => int(c.trim()))
.filter((c): c is number => c !== undefined)
const fallback = SEARCH_FUNCTIONS[t]
const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : [])
const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
const offset = int(qs.offset) ?? 0
// ID-only lookups (tvdbid, imdbid, …) aren't supported: an empty result rather than everything.
const q = typeof qs.q === 'string' ? qs.q.trim() : ''
const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
const result = idOnly
? null
: await SearchService.search({
q,
categories,
readable: true,
notIncomplete: true,
maxAgeDays: int(qs.maxage),
season: int(qs.season),
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
sort: 'newest',
perPage: limit,
offset,
})
const items = (result?.releases ?? []).map((r) => {
const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
return {
id: r.id,
title: r.name,
category: r.category ?? 8010,
size: r.size,
files: r.fileCount,
postedAt: r.postedAt ? new Date(r.postedAt) : null,
poster: r.poster,
groups: r.groups ?? [],
// Passwords come with phase 2 (archive inspection).
password: false,
season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
}
})
return xml(resultsXml(server, items, offset, result?.total ?? 0), 'application/rss+xml')
}
private server(request: HttpContext['request']): NewznabServer {
const proto = request.header('x-forwarded-proto') ?? 'http'
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey: configuredApiKey() }
}
}
+48 -15
View File
@@ -1,27 +1,60 @@
import type { HttpContext } from '@adonisjs/core/http'
import db from '@adonisjs/lucid/services/db'
import NzbService from '#services/nzb_service'
import SearchService, { type SortOrder } from '#services/search_service'
import { expandCategories } from '#lib/newznab/index'
/** The catalog (REBUILD.md, part 9): a name search and a release's NZB. */
const flag = (value: unknown) => value === '1' || value === 'true' || value === true
/** A size in MB from the query string, as bytes. */
function megabytes(value: unknown): number | undefined {
const mb = Number(value)
return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
}
/** The catalog for the web interface (REBUILD.md, part 9). */
export default class ReleasesController {
/** Releases whose name has every word of `q` (dots, dashes, underscores and spaces alike), newest first. */
/** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
async index({ request }: HttpContext) {
const q = String(request.input('q', '')).trim()
const limit = Math.min(Number(request.input('limit', 50)) || 50, 200)
const query = db
.from('releases')
.select('id', 'name', 'poster', 'groups', 'posted_at', 'files', 'file_total', 'bytes', 'complete', 'readable')
.orderBy('posted_at', 'desc')
.limit(limit)
for (const word of q.split(/[\s._-]+/).filter(Boolean)) query.whereILike('name', `%${word.replace(/[\\%_]/g, '\\$&')}%`)
return { releases: await query }
const qs = request.qs()
const categories = String(qs.cat ?? '')
.split(',')
.map((c) => Number(c.trim()))
.filter((c) => Number.isInteger(c) && c > 0)
return SearchService.search({
q: typeof qs.q === 'string' ? qs.q : undefined,
categories: categories.length ? expandCategories(categories) : undefined,
readable: flag(qs.readable),
complete: flag(qs.complete),
source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
minSize: megabytes(qs.min_size),
maxSize: megabytes(qs.max_size),
sort: qs.sort as SortOrder | undefined,
page: Number(qs.page) || 1,
perPage: Number(qs.per_page) || 50,
// The page shows "1,000+"; counting further made filter clicks slow.
countLimit: 1000,
})
}
/** GET /api/v1/releases/:id, with its files */
async show({ params, response }: HttpContext) {
const release = await SearchService.release(Number(params.id))
return release ?? response.notFound({ error: 'Release not found' })
}
/** GET /api/v1/releases/:id/nzb */
async nzb({ params, response }: HttpContext) {
const nzb = await NzbService.build(Number(params.id))
if (!nzb) return response.notFound({ error: 'No such release' })
if (!nzb) return response.notFound({ error: 'Release not found' })
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
response.header('Content-Type', 'application/x-nzb')
response.header('Content-Disposition', `attachment; filename="${nzb.name.replace(/["\\/]/g, '_')}.nzb"`)
return nzb.xml
response.header('Content-Disposition', `attachment; filename="${filename}"`)
return response.send(nzb.xml)
}
/** GET /api/v1/stats */
async stats() {
return SearchService.stats()
}
}
+20 -17
View File
@@ -1,31 +1,34 @@
import db from '@adonisjs/lucid/services/db'
import { decodeChunk } from '#lib/chunks/index'
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
import { buildNzb } from '#lib/nzb/index'
/**
* A release's NZB (REBUILD.md, part 9), from its chunks in the chunk store:
* every chunk decoded, its files in name order. A file stored twice (the same
* name and first article, e.g. a range read again) is taken once.
* A release's files and NZB (REBUILD.md, part 9), from its chunks in the
* chunk store: every chunk decoded, its files in name order. A file stored
* twice (the same name and first article, e.g. a range read again) is taken
* once.
*/
class NzbService {
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
const release = await db.from('releases').where('id', releaseId).select('name').first()
if (!release) return null
async files(releaseId: number): Promise<ChunkFile[]> {
const chunks = await db.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
const files = new Map<string, ReturnType<typeof decodeChunk>[number]>()
const files = new Map<string, ChunkFile>()
for (const chunk of chunks) {
for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
}
return [...files.values()].sort((a, b) => a.name.localeCompare(b.name))
}
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
const release = await db.from('releases').where('id', releaseId).select('name').first()
if (!release) return null
const xml = buildNzb(
[...files.values()]
.sort((a, b) => a.name.localeCompare(b.name))
.map((f) => ({
name: f.name,
poster: f.poster,
date: f.postedAt ?? 0,
groups: f.groups,
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
})),
(await this.files(releaseId)).map((f) => ({
name: f.name,
poster: f.poster,
date: f.postedAt ?? 0,
groups: f.groups,
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
})),
{ title: release.name }
)
return { name: release.name, xml }
+246
View File
@@ -0,0 +1,246 @@
import db from '@adonisjs/lucid/services/db'
import NzbService from '#services/nzb_service'
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
export interface SearchParams {
q?: string
/** Only releases with a file whose name is readable (not an obfuscated token). */
readable?: boolean
/** Only releases known to have all their files. */
complete?: boolean
/** Leave out releases known to miss files (Newznab). */
notIncomplete?: boolean
/** How the release was recognised: stem, subject or post. */
source?: string
poster?: string
/** Exact category ids (already expanded from parents). */
categories?: number[]
maxAgeDays?: number
/** TV: season and episode, matched as S01E02 or 1x02 in the name. */
season?: number
episode?: number
/** Size bounds in bytes. */
minSize?: number
maxSize?: number
sort?: SortOrder
page?: number
perPage?: number
/** Overrides `page` (Newznab pages by offset). */
offset?: number
/** Count matches up to this many past the page; `total` is capped there. */
countLimit?: number
}
/** A release in search results. */
export interface ReleaseSummary {
id: number
name: string
source: string
poster: string | null
groups: string[]
postedAt: Date | null
fileTotal: number | null
fileCount: number
size: number
readable: boolean
complete: boolean | null
category: number
settled: boolean
}
const SORTS: Record<SortOrder, string> = {
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
largest: 'r.bytes DESC, r.id DESC',
files: 'r.files DESC, r.id DESC',
name: 'r.name ASC, r.id ASC',
}
/** A broad search needn't count every match (each counted row is a heap read). */
const COUNT_LIMIT = 10_000
const STATS_TTL_MS = 60 * 1000
/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
function contains(word: string): string {
return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
}
/** Searching and browsing the catalog, for the web interface and Newznab. */
class SearchService {
async search(params: SearchParams) {
const where: string[] = ['r.files > 0']
const bindings: unknown[] = []
// Every word in the name, through its trigram index; "The.Block.AU" and "the block au" alike.
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
where.push('r.name ILIKE ?')
bindings.push(contains(word))
}
if (params.readable) where.push('r.readable')
if (params.complete) where.push('r.complete')
if (params.notIncomplete) where.push('r.complete IS NOT FALSE')
if (params.source) {
where.push('r.source = ?')
bindings.push(params.source)
}
if (params.poster) {
where.push('r.poster ILIKE ?')
bindings.push(contains(params.poster))
}
if (params.categories?.length) {
where.push('r.category = ANY(?)')
bindings.push(params.categories)
}
if (params.maxAgeDays) {
where.push('r.posted_at >= now() - make_interval(days => ?)')
bindings.push(params.maxAgeDays)
}
if (params.minSize !== undefined) {
where.push('r.bytes >= ?')
bindings.push(params.minSize)
}
if (params.maxSize !== undefined) {
where.push('r.bytes <= ?')
bindings.push(params.maxSize)
}
if (params.season !== undefined) {
const s = params.season
const e = params.episode
where.push('r.name ~* ?')
bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
}
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
const page = Math.max(params.page ?? 1, 1)
const offset = params.offset ?? (page - 1) * perPage
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
const whereSql = where.join(' AND ')
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
const [{ rows }, { rows: counted }] = await Promise.all([
db.rawQuery(
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at
FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`,
[...bindings, perPage, offset]
),
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
])
return {
page,
perPage,
offset,
total: Math.min(Number(counted[0].total), countLimit),
/** More than `total` match (the count stops at `countLimit`). */
totalCapped: Number(counted[0].total) > countLimit,
releases: (rows as any[]).map((r): ReleaseSummary => ({
id: Number(r.id),
name: r.name,
source: r.source,
poster: r.poster,
groups: r.groups,
postedAt: r.posted_at,
fileTotal: r.file_total,
fileCount: Number(r.files),
size: Number(r.bytes),
readable: r.readable,
complete: r.complete,
category: r.category,
settled: r.settled_at !== null,
})),
}
}
/** A release with its files, read from its chunks. */
async release(id: number) {
const r = await db.from('releases').where('id', id).first()
if (!r) return null
const files = await NzbService.files(id)
return {
id: Number(r.id),
name: r.name,
source: r.source,
poster: r.poster,
groups: r.groups,
postedAt: r.posted_at,
fileTotal: r.file_total,
fileCount: Number(r.files),
size: Number(r.bytes),
segments: Number(r.segments),
readable: r.readable,
complete: r.complete,
hasPar2: r.has_par2,
category: r.category,
firstSeenAt: r.first_seen_at,
lastSeenAt: r.last_seen_at,
settledAt: r.settled_at,
files: files.map((f) => ({
name: f.name,
subject: f.subject,
poster: f.poster,
postedAt: f.postedAt === null ? null : new Date(f.postedAt),
size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
parts: f.segments.length,
})),
}
}
/** Totals for the web interface, computed at most once a minute (callers in between share them). */
async stats() {
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
this.statsRefresh ??= this.computeStats()
.then((value) => {
this.statsCache = { at: Date.now(), value }
})
.catch((error) => {
if (!this.statsCache) throw error
})
.finally(() => {
this.statsRefresh = undefined
})
if (!this.statsCache) await this.statsRefresh
}
return this.statsCache!.value
}
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
private statsRefresh?: Promise<void>
/**
* Estimates (a catalog of 100M+ rows can't be counted on every visit):
* the row count from Postgres's statistics, below a million rows exact;
* readable share, files and size from a sample; ranges counted exactly
* (a few hundred thousand rows).
*/
private async computeStats() {
const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
const estimate = Number(rows[0]?.n ?? 0)
const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM (0.5)'
const { rows: s } = await db.rawQuery(
`SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
FROM releases ${sample} WHERE files > 0`
)
const sampled = Number(s[0].sampled)
const releases = sample ? Math.max(estimate, 0) : sampled
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
const { rows: ranges } = await db.rawQuery(
`SELECT count(*) FILTER (WHERE status = 'done') AS done, count(*) AS total,
count(*) FILTER (WHERE status = 'done' AND priority >= 0) AS history_done, count(*) FILTER (WHERE priority >= 0) AS history,
max(done_at) AS last_done_at,
max(done_at) FILTER (WHERE priority < 0) AS tail_done_at
FROM ranges`
)
const g = ranges[0]
return {
approximate: sample !== '',
releases,
readableReleases: scale(s[0].readable),
files: scale(s[0].files),
size: scale(s[0].bytes),
ranges: { done: Number(g.done), total: Number(g.total), historyDone: Number(g.history_done), history: Number(g.history) },
lastIndexedAt: g.last_done_at,
tailIndexedAt: g.tail_done_at,
}
}
}
export default new SearchService()
+17
View File
@@ -0,0 +1,17 @@
import { defineConfig } from '@adonisjs/static'
/**
* Configuration options to tweak the static files middleware.
* The complete set of options are documented on the
* official documentation website.
*
* https://docs.adonisjs.com/guides/static-assets
*/
const staticServerConfig = defineConfig({
enabled: true,
etag: true,
lastModified: true,
dotFiles: 'ignore',
})
export default staticServerConfig
@@ -0,0 +1,39 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
/**
* A release's Newznab category from its name (REBUILD.md, part 9): voyager
* 1's release_category() without the archive contents (phase 2 has them).
* TV (5030/5040/5045), movies (2030/2040/2045), audio (3010/3040),
* books (7020), otherwise 8010.
*/
export default class extends BaseSchema {
async up() {
this.schema.raw(String.raw`
CREATE OR REPLACE FUNCTION release_category(name text) RETURNS integer IMMUTABLE LANGUAGE sql AS $$
SELECT CASE
-- TV: S01E02, 1x02, or a season pack (S01).
WHEN name ~* '(^|[._ -])(S\d{1,2}E\d{1,3}|\d{1,2}x\d{2})([._ -]|$)' OR name ~* '[._ -]S\d{1,2}([._ -]|$)' THEN
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045
WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END
-- Movies: a year plus a source or quality marker.
WHEN name ~* '(^|[._ (-])(19|20)\d{2}([._ )-]|$)'
AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\.?26[45]|hevc|remux)' THEN
CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045
WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END
-- Audio: lossless, mp3, or scene music names (-WEB-2026-GROUP).
WHEN name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040
WHEN name ~* '(^|[._ -])mp3([._ -]|$)' OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\d{2}-[A-Za-z0-9]+$' THEN 3010
WHEN name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020
ELSE 8010
END
$$`)
this.schema.raw(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS category integer GENERATED ALWAYS AS (release_category(name)) STORED`)
// Newznab and the web interface list a category newest first.
this.schema.raw(`CREATE INDEX IF NOT EXISTS releases_category_posted_idx ON releases (category, posted_at DESC)`)
}
async down() {
this.schema.raw(`ALTER TABLE releases DROP COLUMN IF EXISTS category`)
this.schema.raw(`DROP FUNCTION IF EXISTS release_category(text)`)
}
}
+40
View File
@@ -0,0 +1,40 @@
/** Newznab categories this indexer assigns (release_category() in SQL, migration …1200). */
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
{ id: 4000, name: 'PC', subcats: [] },
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
]
/** Parent category of a (sub)category, e.g. 5040 → 5000. */
export function parentCategory(id: number): number {
return Math.floor(id / 1000) * 1000
}
/** "TV > HD" style label. */
export function categoryLabel(id: number): string {
const parent = CATEGORIES.find((c) => c.id === parentCategory(id))
if (!parent) return 'Other'
const sub = parent.subcats.find((s) => s.id === id)
return sub ? `${parent.name} > ${sub.name}` : parent.name
}
/**
* Expands requested categories (`cat=5000,2040`) to the concrete ones to
* match: a parent includes all of its subcategories.
*/
export function expandCategories(requested: number[]): number[] {
const result = new Set<number>()
for (const id of requested) {
const parent = CATEGORIES.find((c) => c.id === id)
if (parent) {
result.add(parent.id)
for (const sub of parent.subcats) result.add(sub.id)
} else {
result.add(id)
}
}
return [...result]
}
+5
View File
@@ -0,0 +1,5 @@
/**
* Newznab API formatting, independent of AdonisJS. From voyager 1 (its CONCEPT.md D23).
*/
export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js'
export { capsXml, errorXml, resultsXml, type NewznabItem, type NewznabServer } from './xml.js'
+107
View File
@@ -0,0 +1,107 @@
import { create } from 'xmlbuilder2'
import { CATEGORIES, categoryLabel, parentCategory } from './categories.js'
const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/'
export interface NewznabItem {
id: number
title: string
category: number
size: number
files: number
postedAt: Date | null
poster: string | null
groups: string[]
password: boolean
/** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */
description?: string
season?: string
episode?: string
}
export interface NewznabServer {
title: string
/** Absolute base URL, e.g. http://homelab:3333 */
baseUrl: string
/** Appended to download links so a client can fetch without extra config. */
apiKey?: string
}
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
export function ubbToText(ubb: string): string {
return ubb
.replace(/<br\s*\/?>|\[br\]/gi, '\n')
.replace(/<\/?[a-z][^>]*>/gi, '')
.replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim()))
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '')
.trim()
}
export function errorXml(code: number, description: string): string {
return create({ version: '1.0', encoding: 'UTF-8' })
.ele('error', { code: String(code), description })
.end({ prettyPrint: true })
}
export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string {
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
caps.ele('registration', { available: 'no', open: 'no' })
const searching = caps.ele('searching')
searching.ele('search', { available: 'yes', supportedParams: 'q' })
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
searching.ele('movie-search', { available: 'yes', supportedParams: 'q' })
searching.ele('audio-search', { available: 'yes', supportedParams: 'q' })
searching.ele('book-search', { available: 'yes', supportedParams: 'q' })
const categories = caps.ele('categories')
for (const category of CATEGORIES) {
const element = categories.ele('category', { id: String(category.id), name: category.name })
for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name })
}
return caps.end({ prettyPrint: true })
}
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string {
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
'version': '2.0',
'xmlns:atom': 'http://www.w3.org/2005/Atom',
'xmlns:newznab': NEWZNAB_NS,
})
const channel = rss.ele('channel')
channel.ele('title').txt(server.title)
channel.ele('description').txt(`${server.title} API results`)
channel.ele('link').txt(`${server.baseUrl}/`)
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
for (const item of items) {
const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}`
const pubDate = (item.postedAt ?? new Date()).toUTCString()
const element = channel.ele('item')
element.ele('title').txt(item.title)
element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id))
element.ele('link').txt(getUrl)
element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
element.ele('pubDate').txt(pubDate)
element.ele('category').txt(categoryLabel(item.category))
element.ele('description').txt(item.description ? ubbToText(item.description) : item.title)
element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
const attr = (name: string, value: string | number) =>
element.ele('newznab:attr', { name, value: String(value) })
attr('category', parentCategory(item.category))
if (item.category !== parentCategory(item.category)) attr('category', item.category)
attr('size', item.size)
attr('files', item.files)
attr('grabs', 0)
attr('password', item.password ? 1 : 0)
if (item.poster) attr('poster', item.poster)
if (item.groups.length) attr('group', item.groups[0])
attr('usenetdate', pubDate)
if (item.season) attr('season', item.season)
if (item.episode) attr('episode', item.episode)
}
return rss.end({ prettyPrint: true })
}
+104 -30
View File
@@ -15,6 +15,7 @@
"@adonisjs/lucid": "^22.4.2",
"@adonisjs/session": "^8.1.0",
"@adonisjs/shield": "^9.0.0",
"@adonisjs/static": "^2.0.1",
"@japa/api-client": "^3.2.1",
"@tuyau/core": "^1.2.2",
"@vinejs/vine": "^4.4.0",
@@ -700,6 +701,27 @@
}
}
},
"node_modules/@adonisjs/static": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@adonisjs/static/-/static-2.0.1.tgz",
"integrity": "sha512-pXSyQgha1yfyQDXPp0ywv7hJqk4JgCCkzWLkku2PW3Y6i/9GUTyzzNTCpz29loZzSF0b/EpiHXQE/sqBhW+p7w==",
"license": "MIT",
"dependencies": {
"serve-static": "^2.2.1"
},
"engines": {
"node": ">=24.0.0"
},
"peerDependencies": {
"@adonisjs/assembler": "^8.0.0-next.7 || ^8.0.0",
"@adonisjs/core": "^7.0.0-next.0 || ^7.0.0"
},
"peerDependenciesMeta": {
"@adonisjs/assembler": {
"optional": true
}
}
},
"node_modules/@adonisjs/tsconfig": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@adonisjs/tsconfig/-/tsconfig-2.0.0.tgz",
@@ -790,9 +812,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -809,9 +828,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -828,9 +844,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -847,9 +860,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -1882,9 +1892,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@@ -1902,9 +1909,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@@ -1922,9 +1926,6 @@
"ppc64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@@ -1942,9 +1943,6 @@
"s390x"
],
"dev": true,
"libc": [
"glibc"
],
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@@ -1962,9 +1960,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@@ -1982,9 +1977,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@@ -3543,6 +3535,12 @@
"node": ">=6"
}
},
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
"license": "MIT"
},
"node_modules/escape-string-regexp": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@@ -5548,6 +5546,15 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/path-browserify": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz",
@@ -6078,6 +6085,19 @@
"node": ">= 0.8"
}
},
"node_modules/range-parser": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz",
"integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/raw-body": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
@@ -6375,6 +6395,60 @@
"node": ">=10"
}
},
"node_modules/send": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz",
"integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.3",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"fresh": "^2.0.0",
"http-errors": "^2.0.1",
"mime-types": "^3.0.2",
"ms": "^2.1.3",
"on-finished": "^2.4.1",
"range-parser": "^1.2.1",
"statuses": "^2.0.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/send/node_modules/fresh": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz",
"integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/serve-static": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz",
"integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==",
"license": "MIT",
"dependencies": {
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"parseurl": "^1.3.3",
"send": "^1.2.0"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/set-cookie-parser": {
"version": "2.7.2",
"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz",
+1
View File
@@ -63,6 +63,7 @@
"@adonisjs/lucid": "^22.4.2",
"@adonisjs/session": "^8.1.0",
"@adonisjs/shield": "^9.0.0",
"@adonisjs/static": "^2.0.1",
"@japa/api-client": "^3.2.1",
"@tuyau/core": "^1.2.2",
"@vinejs/vine": "^4.4.0",
+3
View File
@@ -45,6 +45,9 @@ export default await Env.create(new URL('../', import.meta.url), {
NNTP_CONNECTIONS: Env.schema.number.optional(),
NNTP_TIMEOUT_MS: Env.schema.number.optional(),
// Newznab API key (empty: no key needed)
NEWZNAB_API_KEY: Env.schema.secret.optional(),
// Coordinator
RANGE_ARTICLES: Env.schema.number.optional(),
TAIL_EVERY_SECONDS: Env.schema.number.optional(),
+1
View File
@@ -26,6 +26,7 @@ server.use([
() => import('#middleware/force_json_response_middleware'),
() => import('#middleware/container_bindings_middleware'),
() => import('@adonisjs/cors/cors_middleware'),
() => import('@adonisjs/static/static_middleware')
])
/**
+9 -10
View File
@@ -11,9 +11,10 @@ import { middleware } from '#start/kernel'
import router from '@adonisjs/core/services/router'
import { controllers } from '#generated/controllers'
router.get('/', () => {
return { hello: 'world' }
})
// The web interface is public/index.html (static middleware).
router.get('/health', () => ({ ok: true }))
// Newznab API for Prowlarr, Sonarr, Radarr (from voyager 1).
router.get('/api', [controllers.Newznab, 'handle'])
router
.group(() => {
@@ -34,13 +35,11 @@ router
.as('profile')
.use(middleware.auth())
// The catalog (REBUILD.md, part 9).
router
.group(() => {
router.get('releases', [controllers.Releases, 'index'])
router.get('releases/:id/nzb', [controllers.Releases, 'nzb'])
})
.use(middleware.auth({ guards: ['api'] }))
// The catalog (REBUILD.md, part 9): open, as voyager 1's (the web interface uses it); Newznab has its key.
router.get('releases', [controllers.Releases, 'index'])
router.get('releases/:id', [controllers.Releases, 'show']).where('id', router.matchers.number())
router.get('releases/:id/nzb', [controllers.Releases, 'nzb']).where('id', router.matchers.number())
router.get('stats', [controllers.Releases, 'stats'])
// Edge workers (REBUILD.md, parts 1–2).
router
+37 -6
View File
@@ -1,7 +1,6 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import User from '#models/user'
import BatchStore from '#services/batch_store'
import { RangeIndexer } from '#lib/edge/index'
import { parseNzb } from '#lib/nzb/index'
@@ -29,15 +28,13 @@ test.group('Releases API', (group) => {
indexer.finish()
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
const user = await User.create({ email: 'u@example.invalid', password: 'secret-password' })
const token = (await User.accessTokens.create(user)).value!.release()
;(await client.get('/api/v1/releases?q=show s01e01')).assertStatus(401)
const found = await client.get('/api/v1/releases?q=show s01e01').bearerToken(token)
// Open, as voyager 1's catalog (the web interface uses it).
const found = await client.get('/api/v1/releases?q=show s01e01')
found.assertStatus(200)
const releases = (found.body() as { releases: { id: number; name: string }[] }).releases
assert.lengthOf(releases, 1)
const nzb = await client.get(`/api/v1/releases/${releases[0].id}/nzb`).bearerToken(token)
const nzb = await client.get(`/api/v1/releases/${releases[0].id}/nzb`)
nzb.assertStatus(200)
assert.include(nzb.header('content-type'), 'application/x-nzb')
const parsed = parseNzb(nzb.text())!
@@ -45,4 +42,38 @@ test.group('Releases API', (group) => {
assert.deepEqual(parsed.files.map((f) => f.segments.length), [1, 3])
assert.equal(parsed.files[1].segments[2].id, 'Some.Show.S01E01.1080p.WEB.part1.rar-3@test')
})
test('details with files, categories, stats and the Newznab API', async ({ client, assert }) => {
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
const [n] = await db.table('nodes').insert({ name: 'api-node' }).returning('id')
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: n.id }).returning('id')
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
indexer.add([...file('Some.Show.S01E01.1080p.WEB.mkv', 'Some.Show.S01E01.1080p.WEB [1/1]', 2, 100), ...file('Other.Movie.2024.2160p.BluRay.x265.mkv', 'Other.Movie.2024.2160p.BluRay.x265 [1/1]', 1, 300)], 1001)
indexer.finish()
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
// Web API: a category (TV, expanded to its subcategories) and a release's files.
const tv = (await client.get('/api/v1/releases?cat=5000')).body() as { releases: { id: number; name: string; category: number }[] }
assert.deepEqual(tv.releases.map((x) => [x.name, x.category]), [['Some Show S01E01 1080p WEB', 5040]])
const detail = (await client.get(`/api/v1/releases/${tv.releases[0].id}`)).body() as { files: { name: string; parts: number; size: number }[] }
assert.deepEqual(detail.files, [{ ...detail.files[0], name: 'Some.Show.S01E01.1080p.WEB.mkv', parts: 2, size: 1400001 }])
;(await client.get('/api/v1/releases/999999')).assertStatus(404)
const stats = (await client.get('/api/v1/stats')).body() as { releases: number; ranges: { total: number } }
assert.equal(stats.releases, 2)
assert.equal(stats.ranges.total, 1)
// Newznab: caps, a TV search with season and episode, the NZB; the key when one is set.
const caps = await client.get('/api?t=caps')
assert.include(caps.text(), '<category id="5000" name="TV">')
const search = await client.get('/api?t=tvsearch&q=some show&season=1&ep=1')
assert.include(search.text(), '<title>Some Show S01E01 1080p WEB</title>')
assert.include(search.text(), '<newznab:attr name="category" value="5040"/>')
assert.notInclude(search.text(), 'Other Movie')
const movie = await client.get('/api?t=movie&cat=2045')
assert.include(movie.text(), 'Other Movie 2024 2160p BluRay x265')
const get = await client.get(`/api?t=get&id=${tv.releases[0].id}`)
assert.include(get.header('content-type'), 'application/x-nzb')
assert.equal(parseNzb(get.text())!.files.length, 1)
assert.include((await client.get('/api?t=get&id=999999')).text(), 'code="300"')
assert.include((await client.get('/api?t=nope')).text(), 'code="202"')
})
})