Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/IndexMaintenance.ts

@@ -0,0 +1,85 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Only indexes at least this big are worth checking. */
+const MIN_BYTES = 50 * 1024 * 1024
+/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
+const MAX_LEAF_DENSITY = 50
+
+export interface Rebuilt {
+  index: string
+  density: number
+  before: number
+  after: number
+}
+
+/**
+ * GC deletes millions of rows (single-article binaries, copied parts, empty
+ * releases). Vacuum makes their index entries reusable but doesn't compact
+ * the index, so B-trees on the churning tables end up 10–20% full (LOG.md
+ * 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
+ * D33). Needs the pgstattuple extension.
+ */
+class IndexMaintenance {
+  async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
+    const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
+    if (extension.length === 0) {
+      logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
+      return []
+    }
+    if (!dryRun) await this.dropLeftovers()
+
+    const { rows: candidates } = await db.rawQuery(
+      `SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
+       FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
+       JOIN pg_namespace n ON n.oid = c.relnamespace
+       WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
+       ORDER BY 2 DESC`,
+      [MIN_BYTES]
+    )
+    const rebuilt: Rebuilt[] = []
+    for (const { index, bytes } of candidates) {
+      const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
+      const density = Number(rows[0].avg_leaf_density)
+      if (density >= MAX_LEAF_DENSITY) continue
+      if (!dryRun && !(await this.reindex(index))) continue
+      const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
+      const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
+      rebuilt.push(result)
+      logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
+    }
+    return rebuilt
+  }
+
+  /**
+   * REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
+   * binaries during a backfill). Its half-built copy is dropped and it is
+   * tried once more; a second failure skips the index until the next run.
+   */
+  /** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
+  private async dropLeftovers(index = '') {
+    const { rows } = await db.rawQuery(
+      `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
+       WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
+      [index]
+    )
+    for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
+  }
+
+  private async reindex(index: string): Promise<boolean> {
+    for (let attempt = 1; attempt <= 2; attempt++) {
+      try {
+        await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
+        return true
+      } catch (error: any) {
+        logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
+        await this.dropLeftovers(index)
+      }
+    }
+    return false
+  }
+}
+
+const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
+
+export default new IndexMaintenance()