daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Spotnet import, header backfill, staging and counter grouping, faster search

Browse Source 
Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-03 19:59:23 +02:00
1 parent bcb9be5da9
commit be8a3d4580
97 files changed
+4454 -268
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/NzbImportService.ts
+83 -20
View File
@@ -7,6 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
import { parseSubject } from '#lib/subject/index'
import { fileStem } from '#lib/release/index'
import { looksRandom } from '#services/ArchiveService'
import QueueService from '#services/QueueService'
import FileStore from '#services/FileStore'
import { isArchiveStart } from '#lib/content/index'
export interface NzbImportResult {
releaseId: number
@@ -20,8 +23,29 @@ export interface NzbImportResult {
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
export function nzbFileName(subject: string): string {
const parsed = parseSubject(subject)
const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
return unquotedFileName(subject)
}
/**
* A filename from a subject without quotes, as some spot NZBs have:
* `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
* `06 - Song.flac Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
* trailing size are dropped; then the name ends at the first extension
* followed by a space.
*/
export function unquotedFileName(subject: string): string {
const cleaned = subject
.replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
.replace(/\s\d+\s*$/, ' ')
.replace(/(\s+-)+\s*$/, '')
.replace(/^\s*-\s+/, '')
.replace(/\s+/g, ' ')
.trim()
const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
return name.replace(/\s+-$/, '').trim() || subject.trim()
}
export interface NzbImportOptions {
@@ -36,6 +60,13 @@ export interface NzbImportOptions {
name?: string
/** Release source for new releases (default `nzb`). */
source?: ReleaseIdentity['source']
/**
* The NZB is authoritative for which files form the release and what it is
* called (Spotnet spots): its files get the NZB's own release, named
* `name`, also when name rules had put them in another release. Files
* another trusted NZB claimed first stay where they are.
*/
trusted?: boolean
}
/**
@@ -55,21 +86,39 @@ class NzbImportService {
firstId: f.segments[0].id,
}))
return db.transaction(async (trx) => {
const added: { id: number; filename: string }[] = []
const result = await db.transaction(async (trx) => {
const known = await trx
.from('files')
.whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
.select('id', 'release_id', `message_ids`)
const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
.whereIn('first_message_id', entries.map((e) => e.firstId))
.select('id', 'release_id', 'first_message_id')
const knownIds = new Set(known.map((f) => f.first_message_id))
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
const counts = new Map<number, number>()
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
// Releases of other trusted NZBs: their files are not taken over.
let claimed = new Set<number>()
if (options.trusted) {
// Only while that NZB still stands: a disposed spot's files go to its correction.
const holders = await trx
.from('releases as r')
.whereIn('r.id', [...counts.keys()])
.where('r.source', 'spot')
.whereNot('r.release_key', key)
.whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
.select('r.id')
claimed = new Set(holders.map((r) => Number(r.id)))
// Join another spot's release when it holds most of these files (a repost); else use our own.
if (releaseId && !claimed.has(releaseId)) releaseId = undefined
if (releaseId) claimed.delete(releaseId)
}
if (!releaseId) {
const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
const ids = await IngestService.upsertReleases(trx.knexClient, [
{
release: { key, name: title, source: options.source ?? 'nzb' },
@@ -84,9 +133,8 @@ class NzbImportService {
const now = new Date()
const missing = entries.filter((e) => !knownIds.has(e.firstId))
for (let i = 0; i < missing.length; i += 500) {
await trx
.knexClient('files')
.insert(
added.push(
...(await FileStore.insert(
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
filename: name,
@@ -94,22 +142,23 @@ class NzbImportService {
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
parts: file.segments.length,
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
groups: JSON.stringify(file.groups),
release_id: releaseId,
created_at: now,
updated_at: now,
}))
)
.onConflict('file_key')
.ignore()
})),
trx
))
)
}
// The poster's NZB says which files form the post: known files join this
// release, also when name rules had split them over several.
const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
await trx
.from('files')
.whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
.whereIn('id', moving.map((f) => f.id))
.update({ release_id: releaseId })
if (previous.length) {
await trx.rawQuery(
@@ -130,9 +179,15 @@ class NzbImportService {
const release = await trx.from('releases').where('id', releaseId).first()
const update: Record<string, unknown> = {}
if (nzb.meta.password) update.password = nzb.meta.password
if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
if (!release.file_total) update.file_total = entries.length
if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
if (options.trusted) {
// Only its own release: a repost joining another spot's release keeps that one's title.
if (release.release_key === key && release.name !== title) update.name = title
if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
} else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
update.name = title
}
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
const result = {
@@ -149,6 +204,14 @@ class NzbImportService {
)
return result
})
// Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
const archives = added.filter((f) => isArchiveStart(f.filename))
if (archives.length) {
await QueueService.collectionQueue.addBulk(
archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
)
}
return result
}
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */