Files
Daan MeijerandClaude Opus 5.5 be8a3d4580 Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 19:59:23 +02:00

108 lines
4.6 KiB
TypeScript

import { create } from 'xmlbuilder2'
import { CATEGORIES, categoryLabel, parentCategory } from './categories.js'
const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/'
export interface NewznabItem {
id: number
title: string
category: number
size: number
files: number
postedAt: Date | null
poster: string | null
groups: string[]
password: boolean
/** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */
description?: string
season?: string
episode?: string
}
export interface NewznabServer {
title: string
/** Absolute base URL, e.g. http://homelab:3333 */
baseUrl: string
/** Appended to download links so a client can fetch without extra config. */
apiKey?: string
}
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
export function ubbToText(ubb: string): string {
return ubb
.replace(/<br\s*\/?>|\[br\]/gi, '\n')
.replace(/<\/?[a-z][^>]*>/gi, '')
.replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim()))
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '')
.trim()
}
export function errorXml(code: number, description: string): string {
return create({ version: '1.0', encoding: 'UTF-8' })
.ele('error', { code: String(code), description })
.end({ prettyPrint: true })
}
export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string {
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
caps.ele('registration', { available: 'no', open: 'no' })
const searching = caps.ele('searching')
searching.ele('search', { available: 'yes', supportedParams: 'q' })
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
searching.ele('movie-search', { available: 'yes', supportedParams: 'q' })
searching.ele('audio-search', { available: 'yes', supportedParams: 'q' })
searching.ele('book-search', { available: 'yes', supportedParams: 'q' })
const categories = caps.ele('categories')
for (const category of CATEGORIES) {
const element = categories.ele('category', { id: String(category.id), name: category.name })
for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name })
}
return caps.end({ prettyPrint: true })
}
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string {
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
'version': '2.0',
'xmlns:atom': 'http://www.w3.org/2005/Atom',
'xmlns:newznab': NEWZNAB_NS,
})
const channel = rss.ele('channel')
channel.ele('title').txt(server.title)
channel.ele('description').txt(`${server.title} API results`)
channel.ele('link').txt(`${server.baseUrl}/`)
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
for (const item of items) {
const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}`
const pubDate = (item.postedAt ?? new Date()).toUTCString()
const element = channel.ele('item')
element.ele('title').txt(item.title)
element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id))
element.ele('link').txt(getUrl)
element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
element.ele('pubDate').txt(pubDate)
element.ele('category').txt(categoryLabel(item.category))
element.ele('description').txt(item.description ? ubbToText(item.description) : item.title)
element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
const attr = (name: string, value: string | number) =>
element.ele('newznab:attr', { name, value: String(value) })
attr('category', parentCategory(item.category))
if (item.category !== parentCategory(item.category)) attr('category', item.category)
attr('size', item.size)
attr('files', item.files)
attr('grabs', 0)
attr('password', item.password ? 1 : 0)
if (item.poster) attr('poster', item.poster)
if (item.groups.length) attr('group', item.groups[0])
attr('usenetdate', pubDate)
if (item.season) attr('season', item.season)
if (item.episode) attr('episode', item.episode)
}
return rss.end({ prettyPrint: true })
}