Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/nntp/date_search.ts

@@ -0,0 +1,27 @@
+/**
+ * The article number where a group's posts reach `target` (ms since epoch),
+ * by binary search between `first` and `last`. Article numbers only roughly
+ * follow posting time, so the answer is within `precision` articles of the
+ * first one posted at or after `target`.
+ *
+ * `dateOf(n)` returns the posting time of the first article at or after `n`
+ * that still exists, or null when there is none nearby (expired or removed);
+ * a gap counts as older than `target`.
+ */
+export async function articleAtDate(
+  first: bigint,
+  last: bigint,
+  target: number,
+  dateOf: (n: bigint) => Promise<number | null>,
+  precision = 10_000n
+): Promise<bigint> {
+  let lo = first
+  let hi = last
+  while (hi - lo > precision) {
+    const mid = lo + (hi - lo) / 2n
+    const date = await dateOf(mid)
+    if (date === null || date < target) lo = mid
+    else hi = mid
+  }
+  return lo
+}

lib/nntp/delimited_reader.ts

@@ -0,0 +1,76 @@
+const CRLF = Buffer.from('\r\n')
+
+/**
+ * Splits a byte stream into messages that end in a delimiter, which the
+ * consumer can change per message: `next(delimiter)` returns everything up
+ * to the next `delimiter` (excluded). A replacement for nntp-js's
+ * findByteSequence, which has two bugs:
+ *
+ * - An empty multi-line reply (`224 …\r\n.\r\n`, e.g. XOVER of a range
+ *   without articles) never ends: the status line takes the CRLF that
+ *   `\r\n.\r\n` starts with. Here, when a delimiter that starts with CRLF
+ *   follows one that ended with CRLF, that CRLF counts for both.
+ * - A delimiter split over two socket chunks left its first bytes in the
+ *   returned data.
+ *
+ * Empty messages for a bare CRLF delimiter are skipped, as nntp-js does.
+ */
+export async function* delimitedReader(
+  source: AsyncIterable<Uint8Array>,
+  initial: Uint8Array
+): AsyncGenerator<Buffer, void, Uint8Array | undefined> {
+  let delimiter = Buffer.from(initial)
+  let parts: Buffer[] = []
+  let length = 0
+  // Bytes at the start of the buffer, already searched without a match.
+  let searched = 0
+  // Leading bytes that belong to the previous delimiter (a shared CRLF).
+  let skip = 0
+
+  for await (const chunk of source) {
+    parts.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))
+    length += chunk.byteLength
+
+    while (true) {
+      const from = Math.max(0, searched - (delimiter.length - 1))
+      const found = indexOf(parts, length, delimiter, from)
+      if (found === -1) {
+        searched = length
+        break
+      }
+      const all = parts.length === 1 ? parts[0] : Buffer.concat(parts, length)
+      const message = all.subarray(Math.min(skip, found), found)
+      const rest = all.subarray(found + delimiter.length)
+      const previous = delimiter
+
+      if (message.length === 0 && delimiter.equals(CRLF)) {
+        // A stray empty line: keep looking for the same delimiter.
+      } else {
+        delimiter = Buffer.from((yield message) ?? delimiter)
+      }
+
+      const shareCrlf = endsWithCrlf(previous) && delimiter.length > 2 && startsWithCrlf(delimiter)
+      parts = shareCrlf ? [CRLF, rest] : [rest]
+      length = rest.length + (shareCrlf ? 2 : 0)
+      skip = shareCrlf ? 2 : 0
+      searched = 0
+    }
+  }
+
+  if (length > skip) yield Buffer.concat(parts, length).subarray(skip)
+}
+
+const endsWithCrlf = (b: Buffer) => b.length >= 2 && b[b.length - 2] === 13 && b[b.length - 1] === 10
+const startsWithCrlf = (b: Buffer) => b.length >= 2 && b[0] === 13 && b[1] === 10
+
+/** indexOf over a list of buffers (their total `length`), starting at `from`; copies only the searched tail. */
+function indexOf(parts: Buffer[], length: number, needle: Buffer, from: number): number {
+  if (parts.length === 1) return parts[0].indexOf(needle, from)
+  // Collect the parts that cover [from, length).
+  let start = length
+  let i = parts.length
+  while (i > 0 && start > from) start -= parts[--i].length
+  const window = Buffer.concat(parts.slice(i), length - start)
+  const found = window.indexOf(needle, from - start)
+  return found === -1 ? -1 : start + found
+}

lib/nntp/nntp_js.ts

@@ -0,0 +1,39 @@
+import { NNTP } from 'nntp-js'
+import { delimitedReader } from './delimited_reader.js'
+
+const CRLF = new Uint8Array([13, 10])
+
+/**
+ * nntp-js with its reply reader replaced by `delimitedReader` (see there for
+ * the bugs it fixes; the empty-reply one hung XOVER of empty ranges forever).
+ * The reader is created in `connect`, so that is overridden: the same steps
+ * as nntp-js 1.0.4, with the other reader.
+ */
+const proto = NNTP.prototype as any
+if (!proto.__delimitedReader) {
+  proto.__delimitedReader = true
+  proto.connect = async function (this: any, readermode = false, timeout: number | undefined = undefined, socket?: any) {
+    try {
+      this.sock = socket ?? this._createSocket(timeout)
+      // The first reply is always the one-line welcome.
+      this.byteReader = delimitedReader(this.sock, CRLF)
+      this.welcome = new TextDecoder('utf-8').decode(await this._getresp())
+      this.caps = undefined
+      this.connected = true
+      await this.getcapabilities()
+      this.readermodeAfterauth = false
+      if (readermode && !this.caps.READER) {
+        await this._setreadermode()
+        if (!this.readermodeAfterauth) {
+          this.caps = undefined
+          await this.getcapabilities()
+        }
+      }
+    } catch (error) {
+      this.sock?.end()
+      throw error
+    }
+  }
+}
+
+export { NNTP }