Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/BackfillService.ts

@@ -0,0 +1,104 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import { articleAtDate } from '#lib/nntp/date_search'
+
+/** Articles XOVERed to find a posting date near an article number. */
+const PROBE = 200n
+
+/**
+ * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
+ * where live indexing began back to the article posted N days ago; `tick`
+ * (every scheduler run) queues low-priority, header-only fetch ranges from
+ * that cursor downwards, newest first, while few are waiting and the
+ * database stays within its size limits.
+ */
+class BackfillService {
+  /** Sets the group's backfill to reach back `days` days. Returns its range. */
+  async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
+    const group = await db.from('groups').where('name', groupName).first()
+    if (!group) throw new Error(`Unknown group ${groupName}`)
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
+      const dateOf = async (n: bigint) => {
+        const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
+        const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
+        const date = header ? Date.parse(header.date) : Number.NaN
+        return Number.isNaN(date) ? null : date
+      }
+      const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
+      // Live indexing began a little before the group was added: continue from there.
+      const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
+      const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
+      await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
+      return next > until ? { from: until, to: next } : null
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+
+  /** Stops a group's backfill (queued ranges still run). */
+  async stop(groupName: string) {
+    await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
+  }
+
+  /** Queues the next backfill ranges. Returns how many. */
+  async tick(): Promise<number> {
+    const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
+    const groups = await db
+      .from('groups')
+      .where('active', true)
+      .whereNotNull('backfill_until_id')
+      .whereRaw('backfill_next_id > backfill_until_id')
+      .orderBy('id')
+    if (groups.length === 0) return 0
+
+    const room = await this.room(maxDefaultGb, maxHddGb)
+    if (room !== true) {
+      logger.warn(`Backfill paused: ${room}.`)
+      return 0
+    }
+    const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
+    let free = maxQueued - Number(count)
+    let queued = 0
+    // Round-robin over the groups, one range each per pass.
+    while (free > 0 && groups.length > 0) {
+      for (const group of [...groups]) {
+        if (free <= 0) break
+        const next = BigInt(group.backfill_next_id)
+        const until = BigInt(group.backfill_until_id)
+        const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
+        await FetchRangeService.queue(group, start, next, { backfill: true })
+        group.backfill_next_id = (start - 1n).toString()
+        await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
+        free--
+        queued++
+        if (start - 1n <= until) {
+          groups.splice(groups.indexOf(group), 1)
+          logger.info(`${group.name}: backfill fully queued.`)
+        }
+      }
+    }
+    return queued
+  }
+
+  /** True when both parts of the database are under their limits, else why not. */
+  private async room(maxDefaultGb: number, maxHddGb: number): Promise<true | string> {
+    const { rows } = await db.rawQuery(
+      `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
+    )
+    const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
+    if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
+    if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
+    return true
+  }
+}
+
+export default new BackfillService()