Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/spotnet/xml.ts

@@ -6,18 +6,25 @@ export interface SpotXml {
   subcats: string[]
   size: number
   website: string
+  /** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
+  description: string
+  tag: string
+  /** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
+  image: SpotImage | null
   /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
   nzbSegments: string[]
   created: number | null
 }
 
+export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }
+
 const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
 
+/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
 function text(value: string | undefined): string {
   if (value === undefined) return ''
   const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
-  if (cdata) return cdata[1]
-  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
+  return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
     if (entity[0] === '#') {
       const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
       return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
@@ -37,6 +44,9 @@ export function parseSpotXml(xml: string): SpotXml | null {
   const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
   const nzb = element(posting, 'NZB') ?? ''
   return {
+    description: text(element(posting, 'Description')).trim(),
+    tag: text(element(posting, 'Tag')).trim(),
+    image: parseImage(posting),
     title: text(element(posting, 'Title')).trim(),
     // Newer keys number categories from 1.
     category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
@@ -48,15 +58,52 @@ export function parseSpotXml(xml: string): SpotXml | null {
   }
 }
 
-/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */
+function parseImage(posting: string): SpotImage | null {
+  const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
+  if (!match) return null
+  const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
+  if (segments.length > 0) {
+    const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
+    return { segments, width: size('Width'), height: size('Height') }
+  }
+  const url = text(match[2]).trim()
+  return /^https?:\/\//i.test(url) ? { url } : null
+}
+
+/**
+ * The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
+ * the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
+ */
+export function decodeSpotText(latin1: string): string {
+  const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
+  return utf8.includes('\uFFFD') ? latin1 : utf8
+}
+
+/**
+ * Header lines of a raw HEAD response; X-XML (split over many lines) joined.
+ * Folded headers are unfolded: old (key 1) spots put the From address on a
+ * continuation line.
+ */
 export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
   const headers: Record<string, string> = {}
   let xml = ''
+  let last: string | null = null
+  let lastIsFirst = false
   for (const line of raw.split(/\r?\n/)) {
+    if (/^[ \t]/.test(line) && last) {
+      if (last === 'x-xml') xml += line
+      else if (lastIsFirst) headers[last] += line
+      continue
+    }
     const colon = line.indexOf(':')
-    if (colon <= 0) continue
+    if (colon <= 0) {
+      last = null
+      continue
+    }
     const name = line.slice(0, colon).toLowerCase()
     const value = line.slice(colon + 1).replace(/^ /, '')
+    last = name
+    lastIsFirst = !(name in headers)
     if (name === 'x-xml') xml += value
     else headers[name] ??= value
   }