Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
47 lines
2.1 KiB
TypeScript
47 lines
2.1 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
|
|
/**
|
|
* Spot listings (CONCEPT.md D29): old spots become a release without files
|
|
* (source `spot`), whose NZB is fetched when first downloaded. Search shows
|
|
* releases with files or spot listings, so the partial release indexes
|
|
* cover both. Spots are verified from their overview when fetched.
|
|
*/
|
|
const VISIBLE = `(file_count > 0 OR source = 'spot')`
|
|
const INDEXES: [name: string, definition: string][] = [
|
|
['releases_name_trgm', `USING gin (name gin_trgm_ops)`],
|
|
['releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops)`],
|
|
['releases_posted_at_idx', `(posted_at DESC NULLS LAST)`],
|
|
['releases_size_index', `(size)`],
|
|
['releases_file_count_index', `(file_count)`],
|
|
['releases_complete_idx', `(complete)`],
|
|
['releases_category_idx', `(category)`],
|
|
]
|
|
|
|
export default class extends BaseSchema {
|
|
static disableTransactions = true
|
|
|
|
async up() {
|
|
await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`)
|
|
// worker:spots claims recent spots first, then the history in posting order.
|
|
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
|
|
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
|
|
await this.swap(VISIBLE)
|
|
}
|
|
|
|
async down() {
|
|
await this.swap('file_count > 0')
|
|
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_article_idx`)
|
|
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_posted_idx`)
|
|
await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`)
|
|
}
|
|
|
|
private async swap(predicate: string) {
|
|
for (const [name, definition] of INDEXES) {
|
|
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_next`)
|
|
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_next ON releases ${definition} WHERE ${predicate}`)
|
|
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)
|
|
await this.db.rawQuery(`ALTER INDEX ${name}_next RENAME TO ${name}`)
|
|
}
|
|
}
|
|
}
|