Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/nzb/build.ts

@@ -1,4 +1,5 @@
 import { create } from 'xmlbuilder2'
+import type { Segment } from './segments.js'
 
 export interface NzbFile {
   /** Name the downloader should use; the real name when it is known. */
@@ -7,8 +8,7 @@ export interface NzbFile {
   /** Posting time in milliseconds. */
   date: number
   groups: string[]
-  /** Segment number → message-id (without angle brackets) and article size. */
-  segments: Record<string, { id: string; size: number }>
+  segments: Segment[]
 }
 
 export interface NzbOptions {
@@ -34,20 +34,17 @@ export function buildNzb(files: NzbFile[], options: NzbOptions = {}): string {
   }
 
   for (const file of files) {
-    const numbers = Object.keys(file.segments)
-      .map(Number)
-      .sort((a, b) => a - b)
+    const sorted = [...file.segments].sort((a, b) => a.number - b.number)
     const element = root.ele('file', {
       poster: file.poster,
       date: String(Math.floor(file.date / 1000)),
-      subject: `"${file.name}" yEnc (1/${numbers.length})`,
+      subject: `"${file.name}" yEnc (1/${sorted.length})`,
     })
     const groups = element.ele('groups')
     for (const group of file.groups) groups.ele('group').txt(group)
     const segments = element.ele('segments')
-    for (const number of numbers) {
-      const segment = file.segments[String(number)]
-      segments.ele('segment', { bytes: String(segment.size), number: String(number) }).txt(segment.id)
+    for (const segment of sorted) {
+      segments.ele('segment', { bytes: String(segment.size), number: String(segment.number) }).txt(segment.id)
     }
   }
   return root.end({ prettyPrint: true })

lib/nzb/index.ts

@@ -3,3 +3,4 @@
  */
 export { buildNzb, type NzbFile, type NzbOptions } from './build.js'
 export { parseNzb, type ParsedNzb, type ParsedNzbFile } from './parse.js'
+export type { Segment } from './segments.js'

lib/nzb/segments.ts

@@ -0,0 +1,33 @@
+import { zstdCompressSync, zstdDecompressSync, constants } from 'node:zlib'
+
+/** One article of a posted file: its number, message-id (without <>) and size in bytes. */
+export interface Segment {
+  number: number
+  id: string
+  size: number
+}
+
+const VERSION = 1
+
+/**
+ * Segment lists as stored in `files.segments` (CONCEPT.md D31): a version
+ * byte, then zstd (level 1) of a JSON line with the sizes (and the numbers,
+ * unless they are 1..N) followed by one message-id per line. About 20 bytes
+ * per segment, against ~38 for the pglz-compressed JSON object it replaces:
+ * message-ids are mostly random, so this is close to what's possible.
+ */
+export function packSegments(segments: Segment[]): Buffer {
+  const sorted = [...segments].sort((a, b) => a.number - b.number)
+  const sequential = sorted.every((s, i) => s.number === i + 1)
+  const head = JSON.stringify(sequential ? { s: sorted.map((s) => s.size) } : { s: sorted.map((s) => s.size), n: sorted.map((s) => s.number) })
+  const text = Buffer.from([head, ...sorted.map((s) => s.id)].join('\n'), 'utf8')
+  const packed = zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 1 } })
+  return Buffer.concat([Buffer.from([VERSION]), packed])
+}
+
+export function unpackSegments(data: Buffer): Segment[] {
+  if (data[0] !== VERSION) throw new Error(`Unknown segment list version ${data[0]}`)
+  const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
+  const { s: sizes, n: numbers } = JSON.parse(head) as { s: number[]; n?: number[] }
+  return sizes.map((size, i) => ({ number: numbers?.[i] ?? i + 1, id: ids[i], size }))
+}