Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/nzb_import.spec.ts

@@ -1,5 +1,7 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
+import type { Segment } from '#lib/nzb/index'
 import testUtils from '@adonisjs/core/services/test_utils'
 import NzbImportService, { nzbFileName } from '#services/NzbImportService'
 import SearchService from '#services/SearchService'
@@ -17,13 +19,10 @@ const XML = `<?xml version="1.0" encoding="utf-8"?>
   </file>
 </nzb>`
 
-async function file(fields: Record<string, unknown>) {
+async function file(fields: Record<string, unknown> & { segments: Segment[] }) {
   const now = new Date()
-  const [row] = await db
-    .table('files')
-    .insert({ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields })
-    .returning('*')
-  return row
+  const [{ id }] = await FileStore.insert([{ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }])
+  return db.from('files').where('id', id).first()
 }
 
 test.group('NzbImportService', (group) => {
@@ -32,6 +31,14 @@ test.group('NzbImportService', (group) => {
   test('names entries despite odd quoting', ({ assert }) => {
     assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar')
     assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2')
+    // Spot NZBs without quotes.
+    assert.equal(nzbFileName('Adventure.part0055.rar - [056/175] (1/1)'), 'Adventure.part0055.rar')
+    assert.equal(nzbFileName('LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2 (1/0) (1/0)'), 'LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2')
+    assert.equal(nzbFileName('Reacher.S04.vol001+02.par2 yEnc'), 'Reacher.S04.vol001+02.par2')
+    assert.equal(nzbFileName('06 - A New Land, a New Music.flac  Mehmet Ali Sanlikol - The Rise Up (2020) (01/35)'), '06 - A New Land, a New Music.flac')
+    assert.equal(nzbFileName('[01/10] - Some.Release.2026.part01.rar yEnc (1/50) 35000000'), 'Some.Release.2026.part01.rar')
+    assert.equal(nzbFileName('Mozart Zauberfloete 2026 mp4 720p.zip.001'), 'Mozart Zauberfloete 2026 mp4 720p.zip.001')
+    assert.equal(nzbFileName('LibreOffice_7.6.4_Win_x86-64.msi.vol0000'), 'LibreOffice_7.6.4_Win_x86-64.msi.vol0000')
   })
 
   test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => {
@@ -40,8 +47,8 @@ test.group('NzbImportService', (group) => {
       .table('releases')
       .insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
-    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
-    const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', message_ids: JSON.stringify({ '1': { id: 'nzb-1@x', size: 2000 } }) })
+    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', segments: [{ number: 1, id: 'nzb-1@x', size: 2000 }] })
 
     const result = (await NzbImportService.import(posted, XML))!
     assert.equal(result.releaseId, Number(release.id))
@@ -70,7 +77,7 @@ test.group('NzbImportService', (group) => {
         <groups><group>alt.binaries.x</group></groups>
         <segments><segment bytes="500" number="1">${id}@x</segment></segments>
       </file></nzb>`
-    const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', message_ids: JSON.stringify({ '1': { id: 'rar-1@x', size: 2000 } }) })
+    const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', segments: [{ number: 1, id: 'rar-1@x', size: 2000 }] })
 
     const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))!
     const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))!
@@ -87,11 +94,44 @@ test.group('NzbImportService', (group) => {
       (await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id
     const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT')
     const lonely = await insert('lonely', '.nzb')
-    await file({ filename: 'x.part1.rar', release_id: target, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
-    const posted = await file({ filename: '.nzb', release_id: lonely, message_ids: JSON.stringify({ '1': { id: 'nzb-2@x', size: 2000 } }) })
+    await file({ filename: 'x.part1.rar', release_id: target, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    const posted = await file({ filename: '.nzb', release_id: lonely, segments: [{ number: 1, id: 'nzb-2@x', size: 2000 }] })
 
     await NzbImportService.import(posted, XML)
     assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target))
     assert.notExists(await db.from('releases').where('id', lonely).first())
   })
+
+  test('a trusted NZB (a spot) takes its files into its own release, named after it', async ({ assert }) => {
+    const now = new Date()
+    const insertRelease = async (key: string, name: string, source: string) =>
+      (
+        await db
+          .table('releases')
+          .insert({ release_key: key.repeat(40).slice(0, 40), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
+          .returning('id')
+      )[0].id
+    // Name rules had put the spot's first volume into a season release with another episode.
+    const season = await insertRelease('s', 'zxcvvbgnhgjytygf', 'stem')
+    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: season, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    await file({ filename: 'other.episode.rar', release_id: season, segments: [{ number: 1, id: 'other-1@x', size: 700 }] })
+    const spotFile = { id: null, filename: 'x.nzb', poster: 'Bart', groups: [], date: now.getTime(), first_message_id: 'spot1@spot.net' }
+
+    const result = (await NzbImportService.import(spotFile, XML, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', moveContainer: false, trusted: true }))!
+    assert.notEqual(result.releaseId, Number(season))
+    const own = await db.from('releases').where('id', result.releaseId).first()
+    assert.include(own, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', file_count: 2 })
+    const left = await db.from('files').where('release_id', season).select('filename')
+    assert.deepEqual(left.map((f) => f.filename), ['other.episode.rar'])
+
+    // A par2 or archive name no longer replaces a spot's title.
+    const { isBetterName } = await import('#services/NameService')
+    assert.isTrue(isBetterName('zxcvvbgnhgjytygf', 'Better.Name.S03E285'))
+
+    // Another spot for the same files (a repost) joins the first one's release instead of taking them.
+    const repost = { ...spotFile, first_message_id: 'spot2@spot.net' }
+    const second = (await NzbImportService.import(repost, XML, { name: 'Repost', source: 'spot', moveContainer: false, trusted: true }))!
+    assert.equal(second.releaseId, result.releaseId)
+    assert.equal(second.releaseName, 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264')
+  })
 })