Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/ContentBackfill.ts

@@ -3,6 +3,7 @@ import { CommandOptions } from '@adonisjs/core/types/ace'
 import db from '@adonisjs/lucid/services/db'
 import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
+import { isArchiveStart } from '#lib/content/index'
 
 const BATCH = 1000
 
@@ -28,17 +29,37 @@ export default class ContentBackfill extends BaseCommand {
   @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
   declare archiveNzbs: boolean
 
+  @flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
+  declare sevenZip: boolean
+
+  @flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
+  declare archives: boolean
+
+  @flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
+  declare sfv: boolean
+
+  @flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
+  declare nicovideo: boolean
+
   public async run() {
     let lastId = 0
     let queued = 0
     try {
       while (!this.limit || queued < this.limit) {
         // Files never inspected: no fingerprint on their binary and no archive listing.
-        const query = db
-          .from('files as f')
-          .join('binaries as b', 'b.file_id', 'f.id')
-          .where('f.id', '>', lastId)
-        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
+        const query = db.from('files as f').where('f.id', '>', lastId)
+        // Files from NZBs (spots) have no binary.
+        if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
+        else query.join('binaries as b', 'b.file_id', 'f.id')
+        if (this.nicovideo) {
+          query.whereNull('f.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
+        } else if (this.sfv) {
+          query.whereNull('f.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
+        } else if (this.archives) {
+          query.whereNull('f.content_type').whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
+        } else if (this.sevenZip) {
+          query.whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
+        } else if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
         else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
         const files = await query
           .orderBy('f.id')
@@ -48,14 +69,19 @@ export default class ContentBackfill extends BaseCommand {
         lastId = Number(files.at(-1).id)
 
         const jobs = files
-          .filter((f) => ContentService.isCandidate(f.filename))
+          .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
           .slice(0, this.limit ? this.limit - queued : undefined)
           .map((f) => ({
             name: 'process-collection',
             data: { fileId: Number(f.id) },
             // A new id per run: BullMQ ignores an add() for an id it still knows.
             // Targeted backfills go ahead of whatever is waiting.
-            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
+            opts: {
+              jobId: `content-${f.id}-${Date.now()}`,
+              lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
+              // niconico captures are many (~190k) and only cosmetic: behind everything else.
+              ...(this.nicovideo ? { priority: 10 } : {}),
+            },
           }))
         if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
         queued += jobs.length