Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
4.6 KiB
TypeScript
108 lines
4.6 KiB
TypeScript
import { create } from 'xmlbuilder2'
|
|
import { CATEGORIES, categoryLabel, parentCategory } from './categories.js'
|
|
|
|
const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/'
|
|
|
|
export interface NewznabItem {
|
|
id: number
|
|
title: string
|
|
category: number
|
|
size: number
|
|
files: number
|
|
postedAt: Date | null
|
|
poster: string | null
|
|
groups: string[]
|
|
password: boolean
|
|
/** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */
|
|
description?: string
|
|
season?: string
|
|
episode?: string
|
|
}
|
|
|
|
export interface NewznabServer {
|
|
title: string
|
|
/** Absolute base URL, e.g. http://homelab:3333 */
|
|
baseUrl: string
|
|
/** Appended to download links so a client can fetch without extra config. */
|
|
apiKey?: string
|
|
}
|
|
|
|
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
|
|
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
|
|
export function ubbToText(ubb: string): string {
|
|
return ubb
|
|
.replace(/<br\s*\/?>|\[br\]/gi, '\n')
|
|
.replace(/<\/?[a-z][^>]*>/gi, '')
|
|
.replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim()))
|
|
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '')
|
|
.trim()
|
|
}
|
|
|
|
export function errorXml(code: number, description: string): string {
|
|
return create({ version: '1.0', encoding: 'UTF-8' })
|
|
.ele('error', { code: String(code), description })
|
|
.end({ prettyPrint: true })
|
|
}
|
|
|
|
export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string {
|
|
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
|
|
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
|
|
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
|
|
caps.ele('registration', { available: 'no', open: 'no' })
|
|
const searching = caps.ele('searching')
|
|
searching.ele('search', { available: 'yes', supportedParams: 'q' })
|
|
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
|
|
searching.ele('movie-search', { available: 'yes', supportedParams: 'q' })
|
|
searching.ele('audio-search', { available: 'yes', supportedParams: 'q' })
|
|
searching.ele('book-search', { available: 'yes', supportedParams: 'q' })
|
|
const categories = caps.ele('categories')
|
|
for (const category of CATEGORIES) {
|
|
const element = categories.ele('category', { id: String(category.id), name: category.name })
|
|
for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name })
|
|
}
|
|
return caps.end({ prettyPrint: true })
|
|
}
|
|
|
|
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string {
|
|
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
|
|
'version': '2.0',
|
|
'xmlns:atom': 'http://www.w3.org/2005/Atom',
|
|
'xmlns:newznab': NEWZNAB_NS,
|
|
})
|
|
const channel = rss.ele('channel')
|
|
channel.ele('title').txt(server.title)
|
|
channel.ele('description').txt(`${server.title} API results`)
|
|
channel.ele('link').txt(`${server.baseUrl}/`)
|
|
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
|
|
|
|
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
|
|
for (const item of items) {
|
|
const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}`
|
|
const pubDate = (item.postedAt ?? new Date()).toUTCString()
|
|
const element = channel.ele('item')
|
|
element.ele('title').txt(item.title)
|
|
element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id))
|
|
element.ele('link').txt(getUrl)
|
|
element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
|
|
element.ele('pubDate').txt(pubDate)
|
|
element.ele('category').txt(categoryLabel(item.category))
|
|
element.ele('description').txt(item.description ? ubbToText(item.description) : item.title)
|
|
element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
|
|
|
|
const attr = (name: string, value: string | number) =>
|
|
element.ele('newznab:attr', { name, value: String(value) })
|
|
attr('category', parentCategory(item.category))
|
|
if (item.category !== parentCategory(item.category)) attr('category', item.category)
|
|
attr('size', item.size)
|
|
attr('files', item.files)
|
|
attr('grabs', 0)
|
|
attr('password', item.password ? 1 : 0)
|
|
if (item.poster) attr('poster', item.poster)
|
|
if (item.groups.length) attr('group', item.groups[0])
|
|
attr('usenetdate', pubDate)
|
|
if (item.season) attr('season', item.season)
|
|
if (item.episode) attr('episode', item.episode)
|
|
}
|
|
return rss.end({ prettyPrint: true })
|
|
}
|