Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
64 lines
3.0 KiB
TypeScript
64 lines
3.0 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
import { packSegments, unpackSegments } from '#lib/nzb/segments'
|
|
|
|
const BATCH = 5000
|
|
|
|
/**
|
|
* Segment lists compressed (CONCEPT.md D31): `files.message_ids` (jsonb,
|
|
* ~38 bytes per segment after pglz) becomes `files.segments` (zstd, ~20),
|
|
* plus `first_message_id` for the lookups the old expression index served.
|
|
* Run with the workers stopped; reclaim the space with VACUUM FULL files.
|
|
*/
|
|
export default class extends BaseSchema {
|
|
static disableTransactions = true
|
|
|
|
async up() {
|
|
await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS segments bytea, ADD COLUMN IF NOT EXISTS first_message_id text`)
|
|
let last = 0
|
|
let done = 0
|
|
while (true) {
|
|
const { rows } = await this.db.rawQuery(
|
|
`SELECT id, message_ids FROM files WHERE id > ? AND segments IS NULL ORDER BY id LIMIT ?`,
|
|
[last, BATCH]
|
|
)
|
|
if (rows.length === 0) break
|
|
const ids: number[] = []
|
|
const packed: Buffer[] = []
|
|
const firsts: (string | null)[] = []
|
|
for (const row of rows) {
|
|
const segments = Object.entries((row.message_ids ?? {}) as Record<string, { id: string; size?: number }>)
|
|
.map(([number, s]) => ({ number: Number(number), id: s.id, size: Number(s.size ?? 0) }))
|
|
.sort((a, b) => a.number - b.number)
|
|
ids.push(Number(row.id))
|
|
packed.push(packSegments(segments))
|
|
firsts.push(segments[0]?.id ?? null)
|
|
}
|
|
await this.db.rawQuery(
|
|
`UPDATE files f SET segments = v.segments, first_message_id = v.first
|
|
FROM unnest(?::bigint[], ?::bytea[], ?::text[]) AS v(id, segments, first) WHERE f.id = v.id`,
|
|
[ids, packed, firsts] as any
|
|
)
|
|
last = ids[ids.length - 1]
|
|
done += rows.length
|
|
if (done % 100_000 < BATCH) console.log(`Packed the segments of ${done} files.`)
|
|
}
|
|
await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN segments SET NOT NULL`)
|
|
await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_first_message_id_idx ON files (first_message_id)`)
|
|
await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_first_segment_idx`)
|
|
await this.db.rawQuery(`ALTER TABLE files DROP COLUMN message_ids`)
|
|
}
|
|
|
|
async down() {
|
|
await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS message_ids jsonb`)
|
|
const { rows } = await this.db.rawQuery(`SELECT id, segments FROM files`)
|
|
for (const row of rows) {
|
|
const json = Object.fromEntries(unpackSegments(row.segments).map((s) => [s.number, { id: s.id, size: s.size }]))
|
|
await this.db.rawQuery(`UPDATE files SET message_ids = ? WHERE id = ?`, [JSON.stringify(json), row.id])
|
|
}
|
|
await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN message_ids SET NOT NULL`)
|
|
await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
|
|
await this.db.rawQuery(`DROP INDEX IF EXISTS files_first_message_id_idx`)
|
|
await this.db.rawQuery(`ALTER TABLE files DROP COLUMN segments, DROP COLUMN first_message_id`)
|
|
}
|
|
}
|