Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bcb9be5da9
commit
be8a3d4580
97 files changed
+4454
-268
No files matched your search
@@ -7,7 +7,7 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
|
||||
import { fileStem, rarReleaseIdentity } from '#lib/release/index'
|
||||
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
|
||||
|
||||
/** Stored per file and release; a listing of thousands of files is not useful. */
|
||||
const MAX_ENTRIES = 500
|
||||
@@ -59,7 +59,9 @@ class ArchiveService {
|
||||
update.encrypted = true
|
||||
}
|
||||
const better = releaseNameFromContents(info)
|
||||
const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
const renamed =
|
||||
release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
if (renamed) update.name = renamed
|
||||
if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
@@ -138,7 +140,7 @@ export async function renameFromStoredContents(): Promise<number> {
|
||||
.select('id', 'name', 'source', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
if (!looksRandom(release.name) && release.source !== 'post') continue
|
||||
if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
|
||||
const better = releaseNameFromContents({ entries: release.contents })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
@@ -147,11 +149,27 @@ export async function renameFromStoredContents(): Promise<number> {
|
||||
return renamed
|
||||
}
|
||||
|
||||
/** Obfuscated names: one long token of mixed letters and digits, no separators. */
|
||||
/**
|
||||
* Obfuscated names: one long token of mixed letters and digits, no
|
||||
* separators, and the other generated shapes seen on Usenet (LOG.md
|
||||
* 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export function looksRandom(name: string): boolean {
|
||||
// An unknown short extension (e.g. ".bin") says nothing about the name.
|
||||
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
// VIDEO_TS, Sample, CD1: real words, but no name.
|
||||
if (isGenericName(name)) return true
|
||||
if (/^[0-9a-f]{16,}$/i.test(name)) return true
|
||||
// 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
|
||||
if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
|
||||
// UUIDs
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
|
||||
// 1650777167 (yEncBin), "83111763 n" (rclone uploads)
|
||||
if (/^\d{6,}( n)?$/.test(name)) return true
|
||||
// prvjr7hapd0i1ny14pz4, and rclone's long base32 names
|
||||
if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
|
||||
// wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
|
||||
if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
|
||||
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
|
||||
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
|
||||
|
||||
Reference in new issue
Block a user