daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Spotnet import, header backfill, staging and counter grouping, faster search

Browse Source 
Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-03 19:59:23 +02:00
1 parent bcb9be5da9
commit be8a3d4580
97 files changed
+4454 -268
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/SearchService.ts
+149 -32
View File
@@ -1,6 +1,20 @@
import db from '@adonisjs/lucid/services/db'
import { buildNzb } from '#lib/nzb/index'
import SpotService from '#services/SpotService'
import { segmentsOf } from '#services/FileStore'
import { spotCategoryLabels } from '#lib/spotnet/index'
/**
* `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
* without asking); `missing`: the archive is encrypted and we don't have
* it; `none`: no password needed as far as we know.
*/
export type PasswordState = 'known' | 'missing' | 'none'
export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
if (hasPassword) return 'known'
return encrypted ? 'missing' : 'none'
}
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
@@ -10,7 +24,7 @@ export interface SearchParams {
readable?: boolean
/** Only releases that have all their files. */
complete?: boolean
/** Hide password-protected releases. */
/** Hide releases that need a password we don't have. */
hideEncrypted?: boolean
source?: string
poster?: string
@@ -31,6 +45,8 @@ export interface SearchParams {
hideContainers?: boolean
/** Include incomplete releases that consist of par2 files only (hidden by default). */
includePar2Only?: boolean
/** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
countLimit?: number
}
const SORTS: Record<SortOrder, string> = {
@@ -58,6 +74,24 @@ function pattern(word: string): string {
return `%${escaped.replace(/[._\- ]+/g, '_')}%`
}
/**
* As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
* spaces as one space. `The.Block.AU` and `the block au` search the same.
*/
export function searchNormalize(text: string): string {
return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
}
/** How long the web UI's totals are reused. */
const STATS_TTL_MS = 60 * 1000
/**
* Counting stops here: a broad search needn't count every match. Each
* counted row is a random heap read, so the web UI counts less (it only
* shows "1,000+"); Newznab clients page by this total and keep the default.
*/
const COUNT_LIMIT = 10_000
/**
* Searching and browsing releases for the web interface (CONCEPT.md D22).
*/
@@ -66,21 +100,19 @@ class SearchService {
const where: string[] = [
VISIBLE_SQL,
// Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`,
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
]
const bindings: unknown[] = []
const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8)
// Every word must occur in the release's search text: its name, archive
// listing and file names (CONCEPT.md D33), one trigram index for all.
const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
for (const word of words) {
const p = pattern(word)
// Every word must match the release name, a file name, or the archive contents.
// One trigram-indexed lookup per table, not a subquery per release.
where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ?
OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`)
bindings.push(p, p, p, p)
where.push(`r.search_text LIKE ?`)
bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
}
if (params.readable) where.push('is_readable_name(r.name)')
if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE')
if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
if (params.hideContainers) where.push('r.container IS NULL')
// par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
@@ -121,6 +153,7 @@ class SearchService {
const offset = params.offset ?? (page - 1) * perPage
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
const whereSql = where.join(' AND ')
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
const [{ rows }, { rows: countRows }] = await Promise.all([
db.rawQuery(
@@ -131,19 +164,22 @@ class SearchService {
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
r.complete, ${PAR2_ONLY_SQL} AS par2_only
r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
(SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
FROM releases r WHERE ${whereSql}
ORDER BY ${order} LIMIT ? OFFSET ?`,
[...bindings, perPage, offset]
),
db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings),
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
])
return {
page,
perPage,
offset,
total: Number(countRows[0].total),
total: Math.min(Number(countRows[0].total), countLimit),
/** More than `total` match (the count stops at `countLimit`). */
totalCapped: Number(countRows[0].total) > countLimit,
releases: rows.map((r: any) => ({
id: Number(r.id),
name: r.name,
@@ -163,6 +199,10 @@ class SearchService {
category: r.category,
container: r.container,
par2Only: r.par2_only,
listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
hasPassword: r.has_password,
passwordState: passwordState(r.encrypted, r.has_password),
spotDescription: r.spot_description,
})),
}
}
@@ -193,14 +233,32 @@ class SearchService {
),
])
const spot = release.source === 'spot'
? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first()
: null
const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
const spot = spotRow ? await SpotService.withDetails(spotRow) : null
return {
id: Number(release.id),
name: release.name,
source: release.source,
spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null,
spot: spot
? {
messageId: spot.message_id,
title: spot.title,
tag: spot.tag,
poster: spot.poster,
postedAt: spot.posted_at,
website: spot.website,
description: spot.description,
image: spot.image?.segments
? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
: spot.image?.url
? { external: spot.image.url }
: null,
categories: spotCategoryLabels(spot.category, spot.subcats),
}
: null,
password: release.password ?? null,
passwordSource: release.password_source ?? null,
passwordState: passwordState(release.encrypted, release.password !== null),
listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
poster: release.poster,
groups: release.groups,
@@ -230,6 +288,22 @@ class SearchService {
}
}
/** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
if (!spot) return null
const data = await SpotService.image(spot)
if (!data) return null
const type =
data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
: data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
: data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
: data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
: data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
: null
return type ? { data, type } : null
}
async nzb(id: number): Promise<{ name: string; xml: string } | null> {
let release = await db.from('releases').where('id', id).first()
if (!release) return null
@@ -244,39 +318,82 @@ class SearchService {
.from('files')
.where('release_id', id)
.orderByRaw('coalesce(real_name, filename)')
.select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids')
.select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
const xml = buildNzb(
files.map((f) => ({
name: f.real_name ?? f.filename,
poster: f.poster,
date: Number(f.date),
groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
segments: f.message_ids,
segments: segmentsOf(f),
})),
{ title: release.name, password: release.password ?? undefined }
)
return { name: release.name, xml }
}
/**
* Totals for the web UI header. They count millions of rows (minutes with
* the backfill running), so they are computed at most once per
* STATS_TTL_MS and never twice at a time; callers in between share it.
*/
async stats() {
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
this.statsRefresh ??= this.computeStats()
.then((value) => {
this.statsCache = { at: Date.now(), value }
})
// A failed background refresh keeps the previous totals; only a caller that waits sees the error.
.catch((error) => {
if (!this.statsCache) throw error
})
.finally(() => {
this.statsRefresh = undefined
})
// Only the very first caller waits; later ones get the previous totals meanwhile.
if (!this.statsCache) await this.statsRefresh
}
return this.statsCache!.value
}
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
private statsRefresh?: Promise<void>
/**
* Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
* from Postgres's statistics (visible releases = the entries of an index
* that covers exactly them), ratios and sizes from a 0.5% sample of the
* releases table. Within a few percent once autovacuum has analysed.
*/
private async computeStats() {
const { rows } = await db.rawQuery(`
SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases,
(SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases,
(SELECT count(*) FROM files) AS files,
(SELECT coalesce(sum(size), 0) FROM files) AS size,
(SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases,
WITH sample AS (
SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
(SELECT reltuples FROM pg_class WHERE relname = 'files') AS files,
(SELECT count(*) FROM sample) AS sampled,
(SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
-- Indexed data only: a spot listing's size is what the spot announces.
(SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
(SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
(SELECT max(updated_at) FROM groups) AS last_indexed_at,
(SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC)
FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`)
(SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
const r = rows[0]
const releases = Math.max(0, Number(r.releases))
const sampled = Number(r.sampled)
// Sample counts scaled to all visible releases.
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
return {
releases: Number(r.releases),
readableReleases: Number(r.readable_releases),
files: Number(r.files),
size: Number(r.size),
encryptedReleases: Number(r.encrypted_releases),
approximate: true,
releases,
readableReleases: scale(r.sampled_readable),
files: Math.max(0, Number(r.files)),
size: scale(r.sampled_size),
encryptedReleases: scale(r.sampled_encrypted),
lastIndexedAt: r.last_indexed_at,
sources: r.sources ?? [],
sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
}
}
}