Files
usenet-indexer/app/controllers/newznab_controller.ts
T
Daan MeijerandClaude Opus 5.5 be8a3d4580 Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 19:59:23 +02:00

124 lines
4.5 KiB
TypeScript

import type { HttpContext } from '@adonisjs/core/http'
import env from '#start/env'
import SearchService from '#services/SearchService'
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
const LIMITS = { max: 100, default: 50 }
/** Search functions and the category each defaults to when `cat` is absent. */
const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
'search': undefined,
'tvsearch': 5000,
'tv-search': 5000,
'movie': 2000,
'movie-search': 2000,
'music': 3000,
'audio': 3000,
'book': 7000,
}
/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */
function configuredApiKey(): string | undefined {
const value: any = env.get('NEWZNAB_API_KEY')
const key = typeof value === 'string' ? value : value?.release()
return key || undefined
}
function int(value: unknown): number | undefined {
return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined
}
/**
* Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like:
* GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
*/
export default class NewznabController {
async handle({ request, response }: HttpContext) {
const qs = request.qs()
const t = String(qs.t ?? '')
const server = this.server(request)
const xml = (body: string, type = 'application/xml') => {
response.header('Content-Type', `${type}; charset=utf-8`)
return response.send(body)
}
const apiKey = configuredApiKey()
if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
if (t === 'caps') return xml(capsXml(server, LIMITS))
if (t === 'get') {
const id = int(qs.id)
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
const nzb = await SearchService.nzb(id)
if (!nzb) return xml(errorXml(300, 'No such item'))
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
response.header('Content-Disposition', `attachment; filename="${filename}"`)
return xml(nzb.xml, 'application/x-nzb')
}
if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
const requested = String(qs.cat ?? '')
.split(',')
.map((c) => int(c.trim()))
.filter((c): c is number => c !== undefined)
const fallback = SEARCH_FUNCTIONS[t]
const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : [])
const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
const offset = int(qs.offset) ?? 0
// ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but
// answer with an empty result rather than everything.
const q = typeof qs.q === 'string' ? qs.q.trim() : ''
const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
const result = idOnly
? { total: 0, releases: [] }
: await SearchService.search({
q,
categories,
readable: true,
notIncomplete: true,
hideContainers: true,
maxAgeDays: int(qs.maxage),
season: int(qs.season),
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
sort: 'newest',
perPage: limit,
offset,
})
const items = result.releases.map((r: any) => {
const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
return {
id: r.id,
title: r.name,
category: r.category ?? 8010,
size: r.size,
files: r.fileCount,
postedAt: r.postedAt ? new Date(r.postedAt) : null,
poster: r.poster,
groups: r.groups ?? [],
// 1 only when a password is needed that we can't give; a known one is in our NZB.
password: r.passwordState === 'missing',
description: r.spotDescription ?? undefined,
season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
}
})
return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml')
}
private server(request: HttpContext['request']): NewznabServer {
const proto = request.header('x-forwarded-proto') ?? 'http'
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
return {
title: 'Usenet Indexer',
baseUrl: `${proto}://${host}`,
apiKey: configuredApiKey(),
}
}
}