Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

database/migrations/1768620765040_pack_file_segments.ts

@@ -0,0 +1,63 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import { packSegments, unpackSegments } from '#lib/nzb/segments'
+
+const BATCH = 5000
+
+/**
+ * Segment lists compressed (CONCEPT.md D31): `files.message_ids` (jsonb,
+ * ~38 bytes per segment after pglz) becomes `files.segments` (zstd, ~20),
+ * plus `first_message_id` for the lookups the old expression index served.
+ * Run with the workers stopped; reclaim the space with VACUUM FULL files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS segments bytea, ADD COLUMN IF NOT EXISTS first_message_id text`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await this.db.rawQuery(
+        `SELECT id, message_ids FROM files WHERE id > ? AND segments IS NULL ORDER BY id LIMIT ?`,
+        [last, BATCH]
+      )
+      if (rows.length === 0) break
+      const ids: number[] = []
+      const packed: Buffer[] = []
+      const firsts: (string | null)[] = []
+      for (const row of rows) {
+        const segments = Object.entries((row.message_ids ?? {}) as Record<string, { id: string; size?: number }>)
+          .map(([number, s]) => ({ number: Number(number), id: s.id, size: Number(s.size ?? 0) }))
+          .sort((a, b) => a.number - b.number)
+        ids.push(Number(row.id))
+        packed.push(packSegments(segments))
+        firsts.push(segments[0]?.id ?? null)
+      }
+      await this.db.rawQuery(
+        `UPDATE files f SET segments = v.segments, first_message_id = v.first
+         FROM unnest(?::bigint[], ?::bytea[], ?::text[]) AS v(id, segments, first) WHERE f.id = v.id`,
+        [ids, packed, firsts] as any
+      )
+      last = ids[ids.length - 1]
+      done += rows.length
+      if (done % 100_000 < BATCH) console.log(`Packed the segments of ${done} files.`)
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN segments SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_first_message_id_idx ON files (first_message_id)`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_first_segment_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN message_ids`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS message_ids jsonb`)
+    const { rows } = await this.db.rawQuery(`SELECT id, segments FROM files`)
+    for (const row of rows) {
+      const json = Object.fromEntries(unpackSegments(row.segments).map((s) => [s.number, { id: s.id, size: s.size }]))
+      await this.db.rawQuery(`UPDATE files SET message_ids = ? WHERE id = ?`, [JSON.stringify(json), row.id])
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN message_ids SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
+    await this.db.rawQuery(`DROP INDEX IF EXISTS files_first_message_id_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN segments, DROP COLUMN first_message_id`)
+  }
+}