Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/ContentService.ts

@@ -6,10 +6,25 @@ import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#
 import NameService from '#services/NameService'
 import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
 import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
-import { sniffContentType, type ContentType } from '#lib/content/index'
+import {
+  isArchiveStart,
+  NICOVIDEO_FILE,
+  NICOVIDEO_RELEASE,
+  nicovideoInfo,
+  nicovideoReleaseName,
+  sniffContentType,
+  zstdPrefix,
+  type ContentType,
+  type NicovideoInfo,
+} from '#lib/content/index'
 import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem } from '#lib/release/index'
+import { fileStem, releaseIdentity } from '#lib/release/index'
+import IngestService from '#services/IngestService'
 import { looksLikeFilename } from '#lib/subject/index'
+import { sevenZipVolume } from '#lib/sevenzip/index'
+import SevenZipService from '#services/SevenZipService'
+import SfvService from '#services/SfvService'
+import { segmentsOf } from '#services/FileStore'
 
 const FIRST_CHUNK = 16 * 1024
 
@@ -35,6 +50,12 @@ export interface ContentResult {
   nzb?: NzbImportResult
   /** NZBs read from inside an archive. */
   archiveNzbs?: NzbImportResult[]
+  /** Files an SFV's listing moved into its release. */
+  sfvMoved?: number
+  /** A niconico capture's release name from its API response (D34). */
+  nicovideo?: string
+  /** The real name from the yEnc header, for a file posted under a random subject (D36). */
+  yencName?: string
 }
 
 /**
@@ -48,14 +69,65 @@ class ContentService {
   /** Random or missing names, and named first RAR volumes (for contents and password flag). */
   isCandidate(name: string) {
     if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
+    if (isArchiveStart(name)) return true
     if (/\.nzb$/i.test(name)) return true
+    // SFVs name the files of their release; niconico captures carry their title (D34).
+    if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
+    // Every 7z volume: the set is listed when its last volume is in (see inspect).
+    if (sevenZipVolume(name) !== null) return true
     return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
   }
 
+  /**
+   * Uploads that randomise the subject per article often keep the real file
+   * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
+   * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
+   * leaves its weak one-file release for the release its name gives, where
+   * the other files of the upload gather. Returns the name, or undefined.
+   */
+  private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
+    const current = String(file.real_name ?? file.filename)
+    if (file.real_name || !looksRandom(fileStem(current))) return undefined
+    if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
+
+    const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
+    const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
+    let releaseId = file.release_id
+    if (weak) {
+      const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
+      if (identity) {
+        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
+          { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
+        ])
+        releaseId = ids.get(identity.key) ?? releaseId
+      }
+    }
+    await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
+    if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
+    return yencName
+  }
+
+  /**
+   * Names a niconico capture's release after the video, while it still has
+   * the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
+   * the search text.
+   */
+  private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
+    if (!file.release_id) return undefined
+    const release = await db.from('releases').where('id', file.release_id).first()
+    if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
+    const name = nicovideoReleaseName(info, id)
+    await db
+      .from('releases')
+      .where('id', release.id)
+      .update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
+    return name
+  }
+
   /** Downloads a whole (small) file whose first segment is already decoded. */
-  private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
-    const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
-    const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
+  private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
+    const parts = segmentsOf(file)
+    const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
     if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
     if (parts.length === 1) return first
 
@@ -63,7 +135,7 @@ class ContentService {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      for (const [, part] of parts) {
+      for (const part of parts) {
         const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
         assembled ??= YencFile.fromSegment(segment)
         assembled.add(segment)
@@ -83,7 +155,7 @@ class ContentService {
    * uploads by other posters.
    */
   private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
-    const archive = await this.readWhole(first, file.message_ids)
+    const archive = await this.readWhole(first, file)
     const info = archive && parseRarHeaders(archive)
     if (!archive || !info) return []
     const entries = info.entries.filter(isNzbEntry)
@@ -104,7 +176,13 @@ class ContentService {
     const file = await db.from('files').where('id', fileId).first()
     const binary = await db.from('binaries').where('file_id', fileId).first()
     if (!file || !this.isCandidate(file.filename)) return null
-    const first = file.message_ids?.['1']
+    // A later 7z volume: list the set from its first volume once all volumes are in.
+    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
+      const volumes = await SevenZipService.volumes(file)
+      if (!volumes || volumes[0].archive) return null
+      return this.inspect(Number(volumes[0].id))
+    }
+    const first = segmentsOf(file)[0]
     if (!first?.id) return null
 
     const conn = await NntpService.acquire()
@@ -142,8 +220,9 @@ class ContentService {
     }
 
     const result: ContentResult = { named: 0, contentType }
-    if (contentType === 'nzbe' || contentType === 'rclone') {
-      // An encrypted NZB, or rclone crypt (names and data encrypted): nothing
+    result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
+    if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
+      // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
       // to read, and useless without the key.
       const releaseId = file.release_id ?? binary?.release_id
       if (releaseId) {
@@ -151,14 +230,28 @@ class ContentService {
       }
       return result
     }
+    const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
+    if (nicovideo) {
+      const info = nicovideoInfo(await zstdPrefix(segment.data))
+      if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
+      return result
+    }
+    if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
+      const text = await this.readWhole(segment.data, file)
+      if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
+      return result
+    }
     if (contentType === 'nzb') {
-      const xml = await this.readWhole(segment.data, file.message_ids)
+      const xml = await this.readWhole(segment.data, file)
       if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
       return result
     }
-    const set = await Par2Service.read(segment.data, file.message_ids)
+    const set = await Par2Service.read(segment.data, segmentsOf(file))
     if (set && binary) {
       result.par2 = await Par2Service.attach(binary, set)
+    } else if (contentType === '7z') {
+      const info = await SevenZipService.list(file, segment.data)
+      if (info) result.archive = await ArchiveService.record(file, binary, info)
     } else {
       const info = parseRarHeaders(segment.data)
       if (info) {