Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-03 19:59:23 +02:00
1 parent bcb9be5da9
commit be8a3d4580
97 files changed
+4454 -268

No files matched your search

+22 -4
View File
@@ -7,7 +7,7 @@ import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import IngestService from '#services/IngestService'
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
import { fileStem, rarReleaseIdentity } from '#lib/release/index'
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
/** Stored per file and release; a listing of thousands of files is not useful. */
const MAX_ENTRIES = 500
@@ -59,7 +59,9 @@ class ArchiveService {
update.encrypted = true
}
const better = releaseNameFromContents(info)
const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
// A spot's title is kept (CONCEPT.md D29).
const renamed =
release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
if (renamed) update.name = renamed
if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
@@ -138,7 +140,7 @@ export async function renameFromStoredContents(): Promise<number> {
.select('id', 'name', 'source', 'contents')
let renamed = 0
for (const release of releases) {
if (!looksRandom(release.name) && release.source !== 'post') continue
if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
const better = releaseNameFromContents({ entries: release.contents })
if (!better || better === release.name) continue
await db.from('releases').where('id', release.id).update({ name: better })
@@ -147,11 +149,27 @@ export async function renameFromStoredContents(): Promise<number> {
return renamed
}
/** Obfuscated names: one long token of mixed letters and digits, no separators. */
/**
* Obfuscated names: one long token of mixed letters and digits, no
* separators, and the other generated shapes seen on Usenet (LOG.md
* 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
*/
export function looksRandom(name: string): boolean {
// An unknown short extension (e.g. ".bin") says nothing about the name.
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
// VIDEO_TS, Sample, CD1: real words, but no name.
if (isGenericName(name)) return true
if (/^[0-9a-f]{16,}$/i.test(name)) return true
// 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
// UUIDs
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
// 1650777167 (yEncBin), "83111763 n" (rclone uploads)
if (/^\d{6,}( n)?$/.test(name)) return true
// prvjr7hapd0i1ny14pz4, and rclone's long base32 names
if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
// wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6