Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/FetchRangeService.ts

@@ -1,6 +1,7 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
 
 /** A failed range is retried after this long... */
 const REQUEUE_AFTER_MS = 5 * 60 * 1000
@@ -20,21 +21,22 @@ interface Group {
  * retried instead of silently skipped. See CONCEPT.md (D18).
  */
 class FetchRangeService {
-  async queue(group: Group, startId: bigint, endId: bigint) {
+  /** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
+  async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
     const now = new Date()
     const [row] = await db
       .table('fetch_ranges')
-      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now })
+      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
       .returning(['id', 'requeues'])
-    await this.addJob(group.name, Number(row.id), 0, startId, endId)
+    await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
   }
 
-  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) {
+  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
     await QueueService.nntpFetchQueue.add(
       'fetch-headers',
-      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId },
+      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
       // The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
-      { jobId: this.jobId(rangeId, requeues) }
+      { jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
     )
   }
 
@@ -75,7 +77,7 @@ class FetchRangeService {
         .from('fetch_ranges')
         .where('id', range.id)
         .update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
-      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id))
+      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
       requeued++
       logger.warn(
         `Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
@@ -85,13 +87,14 @@ class FetchRangeService {
   }
 
   /**
-   * Moves `groups.last_indexed_id` up to just below the first range that is
-   * not done yet, and forgets done ranges. Returns the new watermark.
+   * Moves `groups.last_indexed_id` up to just below the first live range
+   * that is not done yet (backfill ranges lie below it and don't count), and
+   * forgets done ranges. Returns the new watermark.
    */
   async advanceWatermark(group: Group): Promise<bigint> {
     const { rows } = await db.rawQuery(
       `UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
-         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'),
+         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
          g.last_queued_id))
        WHERE g.id = ? RETURNING last_indexed_id`,
       [group.id]