Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

public/app.js

@@ -27,6 +27,7 @@ const state = {
   release: null,
 }
 let requestId = 0
+let searchAbort = null
 
 // --- Formatting ------------------------------------------------------------
 
@@ -118,8 +119,8 @@ function syncControls() {
 
 // --- API -------------------------------------------------------------------
 
-async function api(path) {
-  const response = await fetch(path, { headers: { accept: 'application/json' } })
+async function api(path, { signal } = {}) {
+  const response = await fetch(path, { headers: { accept: 'application/json' }, signal })
   if (!response.ok) throw new Error(`${response.status} ${response.statusText}`)
   return response.json()
 }
@@ -139,11 +140,13 @@ async function loadStats() {
   try {
     const stats = await api('/api/v1/stats')
     const readable = stats.releases ? Math.round((100 * stats.readableReleases) / stats.releases) : 0
+    // Estimated from Postgres statistics and a sample (CONCEPT.md D33).
+    const about = stats.approximate ? '≈ ' : ''
     const items = [
-      ['Releases', number.format(stats.releases)],
-      ['Readable', `${readable}%`],
-      ['Files', number.format(stats.files)],
-      ['Size', formatSize(stats.size)],
+      ['Releases', about + number.format(stats.releases)],
+      ['Readable', `${about}${readable}%`],
+      ['Files', about + number.format(stats.files)],
+      ['Size', about + formatSize(stats.size)],
       ['Indexed', formatAge(stats.lastIndexedAt) || '–'],
     ]
     $('#stats').replaceChildren(...items.map(([dt, dd]) => el('div', {}, el('dt', { text: dt }), el('dd', { text: dd }))))
@@ -174,9 +177,14 @@ function releaseRow(release) {
   if (release.par2Only) badges.append(badge('par2 only', 'warn', 'Only the par2 files arrived; the data they describe is missing'))
   if (release.container === 'nzbe') badges.append(badge('Encrypted NZB', 'bad', 'An encrypted NZB container: unusable without its key'))
   else if (release.container === 'rclone') badges.append(badge('Encrypted (rclone)', 'bad', 'rclone crypt: names and contents encrypted, unusable without the key'))
-  else if (release.encrypted) badges.append(badge('Password', 'bad', 'Password-protected archive'))
+  else if (release.container === 'yencbin') badges.append(badge('Encrypted (.bin)', 'bad', 'An encrypted .bin container (yEncBin): unusable without its key'))
+  else if (release.passwordState === 'missing') badges.append(badge('Password needed', 'bad', 'Password-protected archive; the password is not known'))
+  else if (release.passwordState === 'known') {
+    badges.append(badge('Password known', 'good', 'The password is in the NZB; SABnzbd and NZBGet use it to unpack'))
+  }
   const files = `${release.fileCount} ${release.fileCount === 1 ? 'file' : 'files'}`
-  if (release.complete) badges.append(badge(files, 'good', 'Complete'))
+  if (release.listingOnly) badges.append(badge('Spot', 'accent', 'A Spotnet spot: its NZB is fetched from Usenet when you download it'))
+  else if (release.complete) badges.append(badge(files, 'good', 'Complete'))
   else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced'))
   else if (release.listedFiles) {
     badges.append(badge(`${release.listedPresent}/${release.listedFiles} listed`, 'warn', 'Files the par2 set lists: present / total'))
@@ -195,24 +203,37 @@ function releaseRow(release) {
 async function loadResults({ append = false } = {}) {
   const id = ++requestId
   if (!append) state.page = 1
+  // A newer search makes the running one pointless: cancel it.
+  searchAbort?.abort()
+  const abort = (searchAbort = new AbortController())
   $('#more').hidden = true
-  if (!append) $('#summary').textContent = 'Searching…'
+  if (!append) {
+    $('#summary').textContent = 'Searching…'
+    $('#empty').hidden = true
+    // The old rows no longer match the search; grey them out until the new ones arrive.
+    $('#list').classList.add('stale')
+    $('.results').setAttribute('aria-busy', 'true')
+  }
 
   try {
-    const result = await api(`/api/v1/releases?${searchParams(state.page)}`)
+    const result = await api(`/api/v1/releases?${searchParams(state.page)}`, { signal: abort.signal })
     if (id !== requestId) return // a newer search superseded this one
+    $('#list').classList.remove('stale')
+    $('.results').removeAttribute('aria-busy')
     const rows = result.releases.map(releaseRow)
     if (append) $('#list').append(...rows)
     else $('#list').replaceChildren(...rows)
 
     const shown = $('#list').children.length
     $('#summary').textContent =
-      result.total === 0 ? '' : `${number.format(result.total)} ${result.total === 1 ? 'release' : 'releases'}` +
+      result.total === 0 ? '' : `${number.format(result.total)}${result.totalCapped ? '+' : ''} ${result.total === 1 ? 'release' : 'releases'}` +
         (shown < result.total ? ` · showing ${number.format(shown)}` : '')
     $('#empty').hidden = result.total !== 0
-    $('#more').hidden = shown >= result.total
+    $('#more').hidden = shown >= result.total && !result.totalCapped
   } catch (error) {
     if (id !== requestId) return
+    $('#list').classList.remove('stale')
+    $('.results').removeAttribute('aria-busy')
     $('#summary').textContent = `Search failed: ${error.message}`
   }
 }
@@ -237,6 +258,80 @@ function section(title, note, content) {
   return [el('h3', { class: 'section-title' }, title, note ? el('small', { text: note }) : null), content]
 }
 
+/**
+ * Spotnet descriptions use UBB markup: [br], [b], [i], [u], [url=…]…[/url].
+ * Built as DOM nodes (never HTML); links only when they are http(s).
+ */
+function ubb(text) {
+  const root = el('div', { class: 'spot-description' })
+  const stack = [root]
+  const top = () => stack[stack.length - 1]
+  // Old (key 1) spots use HTML instead: line breaks and simple tags become UBB, the rest is dropped.
+  text = text
+    .replace(/<br\s*\/?>/gi, '[br]')
+    .replace(/<(\/?)(b|i|u)>/gi, '[$1$2]')
+    .replace(/<\/?[a-z][^>]*>/gi, '')
+  for (const token of text.split(/(\[\/?[a-z]+(?:=[^\]]*)?\])/i)) {
+    const tag = /^\[(\/?)([a-z]+)(?:=([^\]]*))?\]$/i.exec(token)
+    if (!tag) {
+      if (token) top().append(token)
+      continue
+    }
+    const [, closing, rawName, arg] = tag
+    const name = rawName.toLowerCase()
+    if (name === 'br') {
+      top().append(el('br'))
+    } else if (!closing && ['b', 'i', 'u', 'url'].includes(name)) {
+      const url = name === 'url' ? (arg ?? '').trim() : ''
+      const node = name === 'url'
+        ? /^https?:\/\//i.test(url) ? el('a', { href: url, target: '_blank', rel: 'noopener noreferrer nofollow' }) : el('span')
+        : el({ b: 'strong', i: 'em', u: 'u' }[name])
+      node.dataset.ubb = name
+      top().append(node)
+      stack.push(node)
+    } else if (closing && stack.length > 1 && top().dataset.ubb === name) {
+      stack.pop()
+    }
+    // Other tags ([img], [color=…], …) are dropped.
+  }
+  return root
+}
+
+function spotSection(release) {
+  const spot = release.spot
+  if (!spot) return []
+  const image = spot.image?.url
+    ? el('img', {
+        class: 'spot-image',
+        src: spot.image.url,
+        alt: '',
+        loading: 'lazy',
+        width: spot.image.width ?? undefined,
+        height: spot.image.height ?? undefined,
+        onerror: (event) => event.target.remove(),
+      })
+    : null
+  const categories = spot.categories?.head
+    ? el('dl', { class: 'facts' },
+        fact('Category', spot.categories.head),
+        ...spot.categories.subcats.map((sub) => fact(sub.type, sub.values.join(', '))))
+    : null
+  const links = el('div', { class: 'spot-links' },
+    spot.website && /^https?:\/\//i.test(spot.website)
+      ? el('a', { href: spot.website, target: '_blank', rel: 'noopener noreferrer nofollow', text: spot.website })
+      : null,
+    spot.image?.external
+      ? el('a', { href: spot.image.external, target: '_blank', rel: 'noopener noreferrer nofollow', text: 'Image (external site)' })
+      : null)
+  const note = [spot.poster && `by ${spot.poster}`, spot.tag && spot.tag !== spot.poster && `tag ${spot.tag}`].filter(Boolean).join(' · ')
+  return section('Spot', note, el('div', { class: 'spot' },
+    image,
+    el('div', { class: 'spot-body' },
+      categories,
+      spot.description ? ubb(spot.description) : el('p', { class: 'empty', text: 'No description.' }),
+      links.childElementCount ? links : null)))
+}
+
 function renderDetail(release) {
   $('#detail-name').textContent = release.name
   $('#detail-source').textContent = `Recognised by ${SOURCE_LABELS[release.source] ?? release.source ?? 'filename'}`
@@ -289,10 +384,22 @@ function renderDetail(release) {
   const notices = []
   if (release.container === 'nzbe') {
     notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted NZB container ("nzbe"): an encrypted download index for others, unreadable without its key.' }))
+  } else if (release.container === 'yencbin') {
+    notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted .bin container (from the yEncBin uploader): the data is encrypted with a key that is not part of the post.' }))
   } else if (release.container === 'rclone') {
     notices.push(el('div', { class: 'notice notice-bad', text: 'rclone crypt: file names and contents are encrypted with a password and salt that are not part of the post.' }))
-  } else if (release.encrypted) {
-    notices.push(el('div', { class: 'notice notice-bad', text: 'Password-protected: the archive needs a password that is not part of the post.' }))
+  } else if (release.passwordState === 'missing') {
+    notices.push(el('div', { class: 'notice notice-bad', text: 'Password needed: the archive is password-protected, and neither the post nor its spot says the password.' }))
+  }
+  if (release.password) {
+    notices.push(el('div', { class: 'notice notice-good' },
+      el('div', {}, 'Password ', el('code', { class: 'password', text: release.password }),
+        release.passwordSource === 'spot' ? ' (from the spot\'s description)' : '',
+        ` is included in the NZB; SABnzbd and NZBGet use it to unpack.${release.encrypted == null ? ' Whether the archive needs it is not known yet.' : ''}`)))
+  }
+  if (release.listingOnly) {
+    notices.push(el('div', { class: 'notice' },
+      `A Spotnet spot${release.spot?.poster ? ` by ${release.spot.poster}` : ''}. Its files are read from the spot's NZB on Usenet when you download it; that takes a few seconds the first time.`))
   }
   if (release.pending.binaries > 0) {
     const pct = release.pending.partsTotal ? Math.round((100 * release.pending.partsHave) / release.pending.partsTotal) : 0
@@ -334,6 +441,7 @@ function renderDetail(release) {
     actions,
     ...notices,
     facts,
+    ...spotSection(release),
     ...section('Files', `${release.files.length} posted`, files),
     ...(par2 ? section('par2 file list', `${listedPresent} of ${release.par2Files.length} present`, par2) : []),
     ...(contents ? section('Archive contents', `${release.contents.length} entries`, contents) : [])