Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/GcScheduler.ts

@@ -5,6 +5,9 @@ import db from '@adonisjs/lucid/services/db'
 import gcConfig from '#config/gc'
 import QueueService from '#services/QueueService'
 import RedisService from '#services/RedisService'
+import IndexMaintenance from '#services/IndexMaintenance'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+import StagingService from '#services/StagingService'
 import Shutdown from '#services/Shutdown'
 
 const CLEAN_BATCH = 1000
@@ -63,13 +66,20 @@ export default class GcScheduler extends BaseCommand {
     this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
 
     if (!this.dryRun) {
-      const budget = Date.now() + gcConfig.deleteBudgetMs
-      const parts = await this.dropCopiedParts(budget)
-      const single = await this.deleteSingleArticleBinaries(budget)
-      const releases = await this.deleteEmptyReleases(budget)
-      if (parts || single || releases) {
+      // Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
+      const budget = () => Date.now() + gcConfig.deleteBudgetMs
+      const parts = await this.dropCopiedParts(budget())
+      const single = await this.deleteSingleArticleBinaries(budget())
+      const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
+      const releases = await this.deleteEmptyReleases(budget())
+      const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
+      if (staging.adopted || staging.expired) {
+        this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
+      }
+      if (parts || single || unfingerprinted || releases) {
         this.logger.info(
-          `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.`
+          `Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
+            `${releases} empty releases deleted.`
         )
       }
     }
@@ -83,6 +93,23 @@ export default class GcScheduler extends BaseCommand {
         `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
       )
     }
+
+    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep.
+    const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+
+    await this.rebuildIndexes()
+  }
+
+  /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
+  private async rebuildIndexes() {
+    const key = 'gc:reindex:last'
+    const last = Number((await RedisService.client.get(key)) ?? 0)
+    if (Date.now() - last < gcConfig.reindexIntervalMs) return
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
   }
 
   /**
@@ -151,10 +178,11 @@ export default class GcScheduler extends BaseCommand {
     let total = 0
     while (Date.now() < budget) {
       const result = await db.rawQuery(
-        `DELETE FROM parts WHERE binary_id IN (
+        // Header-derived parts (no per-part CRC) of binaries that are a file now.
+        `DELETE FROM parts WHERE pcrc32 IS NULL AND binary_id IN (
            SELECT b.id FROM binaries b
-           WHERE b.file_id IS NOT NULL AND b.size IS NULL
-             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id)
+           WHERE b.file_id IS NOT NULL
+             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id AND p.pcrc32 IS NULL)
            LIMIT ?)`,
         [500]
       )
@@ -190,6 +218,29 @@ export default class GcScheduler extends BaseCommand {
     return total
   }
 
+  /**
+   * Completed header-derived binaries without a fingerprint, some hours after
+   * completion: their file exists, their parts are gone, and par2 naming needs
+   * a size and MD5 they don't have. Oldest first, via the last_seen_at index.
+   */
+  private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM binaries WHERE id IN (
+           SELECT id FROM binaries
+           WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
+             AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
+           ORDER BY last_seen_at LIMIT ?)`,
+        [cutoff, gcConfig.deleteBatch]
+      )
+      total += result.rowCount
+      if (result.rowCount < gcConfig.deleteBatch) break
+    }
+    return total
+  }
+
   /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
   private async deleteEmptyReleases(budget: number): Promise<number> {
     const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)