Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/backfill.spec.ts

@@ -0,0 +1,38 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import indexerConfig from '#config/indexer'
+import BackfillService from '#services/BackfillService'
+import FetchRangeService from '#services/FetchRangeService'
+
+test.group('BackfillService', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('queues header-only ranges downwards, capped, without holding back the live watermark', async ({ assert }) => {
+    const now = new Date()
+    const batch = Number(indexerConfig.batchSize)
+    const [g] = await db
+      .table('groups')
+      .insert({
+        name: 'alt.binaries.backfilltest', active: true, fetch_bodies: true, last_queued_id: 5_000_000, last_indexed_id: 5_000_000,
+        backfill_until_id: 5_000_000 - 100 * batch, backfill_next_id: 4_900_000, created_at: now, updated_at: now,
+      })
+      .returning('*')
+
+    const queued = await BackfillService.tick()
+    assert.equal(queued, indexerConfig.backfill.maxQueued)
+    const ranges = await db.from('fetch_ranges').where('group_id', g.id).orderBy('end_id', 'desc')
+    assert.isTrue(ranges.every((r) => r.backfill && r.status === 'queued'))
+    assert.equal(Number(ranges[0].end_id), 4_900_000)
+    assert.equal(Number(ranges[0].start_id), 4_900_000 - batch + 1)
+    const after = await db.from('groups').where('id', g.id).first()
+    assert.equal(Number(after.backfill_next_id), 4_900_000 - queued * batch)
+
+    // The cap holds while those wait.
+    assert.equal(await BackfillService.tick(), 0)
+
+    // Backfill ranges don't hold back the live watermark.
+    await db.from('groups').where('id', g.id).update({ last_queued_id: 5_100_000 })
+    assert.equal(await FetchRangeService.advanceWatermark({ id: g.id, name: g.name }), 5_100_000n)
+  })
+})

tests/functional/counter_grouping.spec.ts

@@ -0,0 +1,70 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import StagingService from '#services/StagingService'
+import IngestService, { segmentFromHeader } from '#services/IngestService'
+import { counterFileKey } from '#lib/subject/index'
+
+let n = 0
+/** An article of a random-subject upload: a new random name every time. */
+const randomArticle = (k: number, total = 4, { counter = '[21/62]', date = 'Thu, 01 Oct 2026 12:00:00 GMT', id }: { counter?: string; date?: string; id?: string } = {}) =>
+  segmentFromHeader(
+    { 'subject': `${counter} "Rnd${++n}xQ7kLm2Pz9Wv4Ty8Ab3Cd" yEnc (${k}/${total})`, 'from': 'bob@home.com', 'message-id': `<${id ?? `art${n}`}@nyuu>`, 'date': date, ':bytes': '700000' },
+    'alt.binaries.boneless',
+    k
+  )!
+
+async function ingestBatch(segments: ReturnType<typeof randomArticle>[], counters = true) {
+  const r = await StagingService.filter(segments, { counters })
+  await IngestService.ingest(r.ingest)
+  return r
+}
+
+test.group('Counter grouping', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('articles with random subjects become one file by poster, counter and part count', async ({ assert }) => {
+    const key = counterFileKey(randomArticle(1))!
+    await ingestBatch([randomArticle(1)])
+    assert.isNull(await db.from('binaries').where('file_key', key).first())
+    await ingestBatch([randomArticle(2)])
+    const binary = await db.from('binaries').where('file_key', key).first()
+    assert.isNotNull(binary)
+    // Later articles join the file directly, until it completes.
+    await ingestBatch([randomArticle(3)])
+    await ingestBatch([randomArticle(4)])
+    const parts = await db.from('parts').where('binary_id', binary.id).orderBy('number').select('number')
+    assert.deepEqual(parts.map((p) => p.number), [1, 2, 3, 4])
+    assert.isNotNull((await db.from('binaries').where('id', binary.id).first()).completed_at)
+  })
+
+  test('off by default; another day is another file; a repeated part number is a conflict', async ({ assert }) => {
+    await ingestBatch([randomArticle(1, 9, { counter: '[05/10]' })], false)
+    await ingestBatch([randomArticle(2, 9, { counter: '[05/10]' })], false)
+    assert.isNull(await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[05/10]' }))!).first())
+
+    const other = randomArticle(1, 9, { counter: '[06/10]', date: 'Fri, 02 Oct 2026 12:00:00 GMT' })
+    assert.notEqual(counterFileKey(other), counterFileKey(randomArticle(1, 9, { counter: '[06/10]' })))
+
+    // Two uploads of the same shape on one day: part 1 twice, from different articles.
+    await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
+    await ingestBatch([randomArticle(2, 9, { counter: '[07/10]' })])
+    await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
+    const binary = await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[07/10]' }))!).first()
+    assert.isNotNull(binary.conflicted_at)
+  })
+
+  test('a normal file with a counter stays a normal file', async ({ assert }) => {
+    const normal = (k: number) =>
+      segmentFromHeader(
+        { 'subject': `[07/45] - "Some.Release.part06.rar" yEnc (${k}/3)`, 'from': 'poster@x', 'message-id': `<norm${k}@x>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
+        'alt.binaries.test',
+        k
+      )!
+    await ingestBatch([normal(1)])
+    await ingestBatch([normal(2), normal(3)])
+    const binary = await db.from('binaries').where('name', 'Some.Release.part06.rar').first()
+    assert.equal(binary.file_key, normal(1).fileKey)
+    assert.isNotNull(binary.completed_at)
+  })
+})

tests/functional/names.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
 import testUtils from '@adonisjs/core/services/test_utils'
 import NameService from '#services/NameService'
 
@@ -36,10 +37,9 @@ test.group('NameService', (group) => {
       .table('releases')
       .insert({ release_key: 'w'.repeat(40), name: 'mOzyyxCyYUjAbCdEf12', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
-    const [file] = await db
-      .table('files')
-      .insert({ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', message_ids: '{}', release_id: weak.id, created_at: now, updated_at: now })
-      .returning('id')
+    const [file] = await FileStore.insert([
+      { filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], release_id: weak.id, created_at: now, updated_at: now },
+    ])
     const episode = await binary('ep18', { size: 939_684_767, md5_16k: 'd'.repeat(32), release_id: weak.id, file_id: file.id })
 
     await NameService.resolve([episode])

tests/functional/newznab.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
 import env from '#start/env'
 import testUtils from '@adonisjs/core/services/test_utils'
 
@@ -11,11 +12,11 @@ async function seed(name: string, opts: { fileTotal?: number; files?: number } =
     .insert({ release_key: `nz${++counter}`.padEnd(40, 'n'), name, source: 'stem', poster: 'poster@example.invalid', groups: '["alt.binaries.test"]', file_total: opts.fileTotal ?? null, posted_at: now, first_seen_at: now, last_seen_at: now })
     .returning('id')
   for (let i = 0; i < (opts.files ?? 1); i++) {
-    await db.table('files').insert({
+    await FileStore.insert([{
       filename: `${name}.part${i + 1}.rar`, poster: 'poster@example.invalid', date: now.getTime(), parts: 1, size: 1_000_000,
-      release_id: release.id, groups: '["alt.binaries.test"]', message_ids: JSON.stringify({ '1': { id: `${name}-${i}@test`, size: 1_000_000 } }),
+      release_id: release.id, groups: '["alt.binaries.test"]', segments: [{ number: 1, id: `${name}-${i}@test`, size: 1_000_000 }],
       created_at: now, updated_at: now,
-    })
+    }])
   }
   return Number(release.id)
 }

tests/functional/nzb_import.spec.ts

@@ -1,5 +1,7 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
+import type { Segment } from '#lib/nzb/index'
 import testUtils from '@adonisjs/core/services/test_utils'
 import NzbImportService, { nzbFileName } from '#services/NzbImportService'
 import SearchService from '#services/SearchService'
@@ -17,13 +19,10 @@ const XML = `<?xml version="1.0" encoding="utf-8"?>
   </file>
 </nzb>`
 
-async function file(fields: Record<string, unknown>) {
+async function file(fields: Record<string, unknown> & { segments: Segment[] }) {
   const now = new Date()
-  const [row] = await db
-    .table('files')
-    .insert({ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields })
-    .returning('*')
-  return row
+  const [{ id }] = await FileStore.insert([{ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }])
+  return db.from('files').where('id', id).first()
 }
 
 test.group('NzbImportService', (group) => {
@@ -32,6 +31,14 @@ test.group('NzbImportService', (group) => {
   test('names entries despite odd quoting', ({ assert }) => {
     assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar')
     assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2')
+    // Spot NZBs without quotes.
+    assert.equal(nzbFileName('Adventure.part0055.rar - [056/175] (1/1)'), 'Adventure.part0055.rar')
+    assert.equal(nzbFileName('LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2 (1/0) (1/0)'), 'LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2')
+    assert.equal(nzbFileName('Reacher.S04.vol001+02.par2 yEnc'), 'Reacher.S04.vol001+02.par2')
+    assert.equal(nzbFileName('06 - A New Land, a New Music.flac  Mehmet Ali Sanlikol - The Rise Up (2020) (01/35)'), '06 - A New Land, a New Music.flac')
+    assert.equal(nzbFileName('[01/10] - Some.Release.2026.part01.rar yEnc (1/50) 35000000'), 'Some.Release.2026.part01.rar')
+    assert.equal(nzbFileName('Mozart Zauberfloete 2026 mp4 720p.zip.001'), 'Mozart Zauberfloete 2026 mp4 720p.zip.001')
+    assert.equal(nzbFileName('LibreOffice_7.6.4_Win_x86-64.msi.vol0000'), 'LibreOffice_7.6.4_Win_x86-64.msi.vol0000')
   })
 
   test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => {
@@ -40,8 +47,8 @@ test.group('NzbImportService', (group) => {
       .table('releases')
       .insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
-    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
-    const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', message_ids: JSON.stringify({ '1': { id: 'nzb-1@x', size: 2000 } }) })
+    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', segments: [{ number: 1, id: 'nzb-1@x', size: 2000 }] })
 
     const result = (await NzbImportService.import(posted, XML))!
     assert.equal(result.releaseId, Number(release.id))
@@ -70,7 +77,7 @@ test.group('NzbImportService', (group) => {
         <groups><group>alt.binaries.x</group></groups>
         <segments><segment bytes="500" number="1">${id}@x</segment></segments>
       </file></nzb>`
-    const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', message_ids: JSON.stringify({ '1': { id: 'rar-1@x', size: 2000 } }) })
+    const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', segments: [{ number: 1, id: 'rar-1@x', size: 2000 }] })
 
     const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))!
     const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))!
@@ -87,11 +94,44 @@ test.group('NzbImportService', (group) => {
       (await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id
     const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT')
     const lonely = await insert('lonely', '.nzb')
-    await file({ filename: 'x.part1.rar', release_id: target, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
-    const posted = await file({ filename: '.nzb', release_id: lonely, message_ids: JSON.stringify({ '1': { id: 'nzb-2@x', size: 2000 } }) })
+    await file({ filename: 'x.part1.rar', release_id: target, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    const posted = await file({ filename: '.nzb', release_id: lonely, segments: [{ number: 1, id: 'nzb-2@x', size: 2000 }] })
 
     await NzbImportService.import(posted, XML)
     assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target))
     assert.notExists(await db.from('releases').where('id', lonely).first())
   })
+
+  test('a trusted NZB (a spot) takes its files into its own release, named after it', async ({ assert }) => {
+    const now = new Date()
+    const insertRelease = async (key: string, name: string, source: string) =>
+      (
+        await db
+          .table('releases')
+          .insert({ release_key: key.repeat(40).slice(0, 40), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
+          .returning('id')
+      )[0].id
+    // Name rules had put the spot's first volume into a season release with another episode.
+    const season = await insertRelease('s', 'zxcvvbgnhgjytygf', 'stem')
+    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: season, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    await file({ filename: 'other.episode.rar', release_id: season, segments: [{ number: 1, id: 'other-1@x', size: 700 }] })
+    const spotFile = { id: null, filename: 'x.nzb', poster: 'Bart', groups: [], date: now.getTime(), first_message_id: 'spot1@spot.net' }
+
+    const result = (await NzbImportService.import(spotFile, XML, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', moveContainer: false, trusted: true }))!
+    assert.notEqual(result.releaseId, Number(season))
+    const own = await db.from('releases').where('id', result.releaseId).first()
+    assert.include(own, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', file_count: 2 })
+    const left = await db.from('files').where('release_id', season).select('filename')
+    assert.deepEqual(left.map((f) => f.filename), ['other.episode.rar'])
+
+    // A par2 or archive name no longer replaces a spot's title.
+    const { isBetterName } = await import('#services/NameService')
+    assert.isTrue(isBetterName('zxcvvbgnhgjytygf', 'Better.Name.S03E285'))
+
+    // Another spot for the same files (a repost) joins the first one's release instead of taking them.
+    const repost = { ...spotFile, first_message_id: 'spot2@spot.net' }
+    const second = (await NzbImportService.import(repost, XML, { name: 'Repost', source: 'spot', moveContainer: false, trusted: true }))!
+    assert.equal(second.releaseId, result.releaseId)
+    assert.equal(second.releaseName, 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264')
+  })
 })

tests/functional/random_names.spec.ts

@@ -0,0 +1,27 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import { looksRandom } from '#services/ArchiveService'
+
+// Names seen on homelab (LOG.md 2026-10-01 (3)).
+const RANDOM = [
+  '5d4063b423f57e1eadda1068b0091409', 'mOzyyxCyYUjAbCdEf12', '6327d9fa2d6acf3e9a61fb053359d5e7.1790574045',
+  'e63afdb4-6a1f-417d-aa1b-b14c0ea1b81d', '1650777167', '83111763 n', 'prvjr7hapd0i1ny14pz4', '8huowjcdamwbetufib9f',
+  '071vl0l120ndtjkjkuh34a2g6aqsak750f55dkudeaea8s4bivjm1l32fd3uduk0dab2t82hp553i29nfjvjt3redp', 'wHJ7s71sAGE', 'EmFqTfL8rxV',
+  'VIDEO_TS', 'Sample', 'CD1', 'Disc 2', 'Subs',
+]
+const READABLE = [
+  'Our.America.with.Lisa.Ling.2011.S01E01.Faith.Healers.1080p.DSCP.WEB-DL.DDP2.0.H.264-FUZEER', '00-graham_barham-club_country',
+  '1790274935_nicovideo_jp_watch_sm26602185', 'StarWars1977', 'nyuu_live_roundtrip', 'Makaja_Gonzales-Freestyle-WEB-2021-XTC_iNT',
+  'The Tom and Jerry Show (2014) S02E13 - Slap Happy Birthday', 'Bondgenoten', 'S01E02', '2026', 'Samples.Of.Love.2020.1080p', 'Subs.and.Dubs',
+]
+
+test.group('random names', () => {
+  test('looksRandom and SQL is_readable_name agree', async ({ assert }) => {
+    for (const name of [...RANDOM, ...READABLE]) {
+      const { rows } = await db.rawQuery('SELECT is_readable_name(?) AS readable', [name])
+      const random = RANDOM.includes(name)
+      assert.equal(looksRandom(name), random, `looksRandom(${name})`)
+      assert.equal(rows[0].readable, !random, `is_readable_name(${name})`)
+    }
+  })
+})

tests/functional/release_merge.spec.ts

@@ -0,0 +1,42 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+
+let keys = 0
+async function postRelease(name: string, files: string[], poster: string) {
+  const now = new Date()
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `merge${++keys}`.padEnd(40, 'm'), name, source: 'post', poster, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  await FileStore.insert(
+    files.map((f, i) => ({ filename: f, poster, date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: '[]',
+      segments: [{ number: 1, id: `${name}-${poster}-${i}@x`, size: 10 }], created_at: now, updated_at: now }))
+  )
+  return Number(r.id)
+}
+
+test.group('Split uploads', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('merges one upload posted under a poster per volume, not reposts or other uploads', async ({ assert }) => {
+    const name = 'Masters.of.the.Universe.1987.Theatrical.Cut.2160p.BDRip-GRP'
+    const a = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part001.rar'], 'niSSYNy5zOW2VX@VMuLu_e7Gr.bIx')
+    const b = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part002.rar'], 'MgrZzFLB3D@MeK4_gntYXzh9g.LBQ')
+    const c = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part003.rar'], 'qNGVviOMN2AJrSzeXFZqvW@VzavS9GO.Y0b')
+    // A repost: the same file names again.
+    const repostName = 'Some.Repost.2026.1080p-GRP'
+    const r1 = await postRelease(repostName, ['abc.part01.rar'], 'p1@x')
+    const r2 = await postRelease(repostName, ['abc.part01.rar'], 'p2@x')
+
+    assert.deepEqual(await ReleaseMergeService.mergeSplitPosts({ dryRun: true }), { names: 1, releases: 2 })
+    await ReleaseMergeService.mergeSplitPosts()
+    assert.equal((await db.from('releases').where('id', a).first()).file_count, 3)
+    assert.equal((await db.from('releases').where('id', b).first()).file_count, 0)
+    assert.equal((await db.from('releases').where('id', c).first()).file_count, 0)
+    assert.equal((await db.from('releases').where('id', r1).first()).file_count, 1)
+    assert.equal((await db.from('releases').where('id', r2).first()).file_count, 1)
+  })
+})

tests/functional/search_text.spec.ts

@@ -0,0 +1,41 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import SearchService, { searchNormalize } from '#services/SearchService'
+
+test.group('Search text', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('normalises like SQL does', async ({ assert }) => {
+    assert.equal(searchNormalize(' The.Block.AU_S22E35 - 1080p '), 'the block au s22e35 1080p')
+    const { rows } = await db.rawQuery(`SELECT search_normalize(' The.Block.AU_S22E35 - 1080p ') AS n`)
+    assert.equal(rows[0].n, 'the block au s22e35 1080p')
+  })
+
+  test('follows files and renames, only for visible releases', async ({ assert }) => {
+    const now = new Date()
+    const [r] = await db
+      .table('releases')
+      .insert({ release_key: 's'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7Rt', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+      .returning('*')
+    assert.isNull(r.search_text)
+
+    // A readable file name that the release name lacks joins the search text.
+    await FileStore.insert([
+      { filename: 'Some.Documentary.2026.1080p.WEB.part01.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
+        groups: '[]', segments: [{ number: 1, id: 'st1@x', size: 10 }], created_at: now, updated_at: now },
+      { filename: 'Q7wErTyUiOpAsDfGhJkL9.part02.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
+        groups: '[]', segments: [{ number: 1, id: 'st2@x', size: 10 }], created_at: now, updated_at: now },
+    ])
+    let row = await db.from('releases').where('id', r.id).first()
+    assert.equal(row.search_text, 'xq8r2kdlm3zw91pqa7rt some documentary 2026 1080p web')
+    const found = await SearchService.search({ q: 'documentary 2026.1080p' } as any)
+    assert.isTrue(found.releases.some((x: any) => x.id === Number(r.id)))
+
+    // A rename recomputes it.
+    await db.from('releases').where('id', r.id).update({ name: 'Some.Documentary.2026.1080p.WEB-GRP' })
+    row = await db.from('releases').where('id', r.id).first()
+    assert.equal(row.search_text, 'some documentary 2026 1080p web grp some documentary 2026 1080p web')
+  })
+})

tests/functional/sfv.spec.ts

@@ -0,0 +1,59 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import { parseSfv } from '#lib/content/index'
+
+const POSTER = 'Snazzy Poster <me@localhost.localdomain>'
+
+let keys = 0
+async function release(name: string, source = 'stem') {
+  const now = new Date()
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `sfv${++keys}`.padEnd(40, 'x'), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  return Number(r.id)
+}
+
+async function file(name: string, releaseId: number, poster = POSTER) {
+  const [{ id }] = await FileStore.insert([
+    { filename: name, poster, date: Date.now(), parts: 1, size: 10, release_id: releaseId, groups: '[]',
+      segments: [{ number: 1, id: `${name}@x`, size: 10 }], created_at: new Date(), updated_at: new Date() },
+  ])
+  return db.from('files').where('id', id).first()
+}
+
+test.group('SFV grouping', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('parses SFV listings', ({ assert }) => {
+    assert.deepEqual(parseSfv('; made by x\r\n01-a-b-4066667a.flac 4066667A\r\n"02 c.flac" 5058F120\r\n\r\n'), [
+      { name: '01-a-b-4066667a.flac', crc: '4066667a' },
+      { name: '02 c.flac', crc: '5058f120' },
+    ])
+  })
+
+  test("gathers an album's tracks into the SFV's release, named after the album; later tracks join too", async ({ assert }) => {
+    const album = await release('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026')
+    const sfv = await file('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026.sfv', album)
+    const track1 = await release('01-graham_barham-country_music-4066667a')
+    await file('01-graham_barham-country_music-4066667a.flac', track1)
+    // Another poster's file with the same name stays.
+    const other = await release('01-graham_barham-country_music-4066667a', 'stem')
+    const foreign = await file('01-graham_barham-country_music-4066667a.flac', other, 'someone@else')
+
+    const listing = '01-graham_barham-country_music-4066667a.flac 4066667a\n02-graham_barham-sippin-5058f120.flac 5058f120\n'
+    assert.equal(await SfvService.attach(sfv, listing), 1)
+    const renamed = await db.from('releases').where('id', album).first()
+    assert.equal(renamed.name, 'graham_barham-club_country_(after_party)-16bit-web-flac-2026')
+    assert.equal((await db.from('files').where('id', foreign.id).first()).release_id, other)
+
+    // Track 2 completes after the SFV was read.
+    const track2 = await release('02-graham_barham-sippin-5058f120')
+    const late = await file('02-graham_barham-sippin-5058f120.flac', track2)
+    assert.equal(await SfvService.claim({ id: late.id, name: late.filename, poster: late.poster, date: Number(late.date), release_id: track2 }), album)
+    assert.equal((await db.from('releases').where('id', album).first()).file_count, 3)
+  })
+})

tests/functional/spot_listing.spec.ts

@@ -0,0 +1,109 @@
+import { test } from '@japa/runner'
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import SpotService, { isDummyNzb } from '#services/SpotService'
+import SearchService from '#services/SearchService'
+
+test.group('SpotService listings', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('old verified spots become visible releases without files', async ({ assert }) => {
+    const now = new Date()
+    const old = new Date('2019-05-01T12:00:00Z')
+    const spot = {
+      message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
+      poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
+    }
+    await db.table('spots').insert(spot)
+    const [claimed] = await SpotService.claim(10)
+    assert.isTrue(SpotService.isListable(claimed))
+    assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
+    assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
+
+    assert.equal(await SpotService.list([claimed]), 1)
+    const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
+    assert.equal(row.status, 'listed')
+    const release = await db.from('releases').where('id', row.release_id).first()
+    assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
+    assert.equal(Number(release.size), 1_500_000_000)
+    // The key an import of this spot's NZB uses, so it fills this release.
+    assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
+
+    const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
+    assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
+  })
+
+  test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
+    // A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
+    const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
+    const overviews = Array.from({ length: 6000 }, (_, i) => [
+      i + 1,
+      { 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
+    ]) as any
+    assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
+    const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
+    assert.equal(Number(count), 6000)
+  })
+
+  test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
+    const spot = {
+      message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
+      poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
+      created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
+      description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
+      details_at: new Date(),
+    }
+    await db.table('spots').insert(spot)
+    const [claimed] = await SpotService.claim(10)
+    await SpotService.list([claimed])
+    const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
+    const release = (await SearchService.release(Number(releaseId)))!
+    assert.equal(release.spot!.description, 'Plot[br]more')
+    assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
+    assert.equal(release.spot!.categories.head, 'Image')
+    assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
+  })
+
+  test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
+    const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
+    assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
+    assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
+    // Text-only spots announce the dummy's own size.
+    assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
+  })
+})
+
+test.group('SpotService passwords', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
+    const now = new Date()
+    await db.table('spots').insert({
+      message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
+      key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
+      description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
+    })
+    const [claimed] = await SpotService.claim(10)
+    await SpotService.list([claimed])
+    assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
+    const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
+    await db.from('releases').where('id', releaseId).update({ encrypted: true })
+    const release = (await SearchService.release(Number(releaseId)))!
+    assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
+
+    // An NZB's own password is not replaced.
+    await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
+    await SpotService.applyPassword('pw1@spot.net')
+    assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
+  })
+
+  test('password states', async ({ assert }) => {
+    const { passwordState } = await import('#services/SearchService')
+    assert.equal(passwordState(true, false), 'missing')
+    assert.equal(passwordState(true, true), 'known')
+    assert.equal(passwordState(null, true), 'known')
+    assert.equal(passwordState(false, false), 'none')
+    assert.equal(passwordState(null, false), 'none')
+  })
+})

tests/functional/staging.spec.ts

@@ -0,0 +1,70 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import StagingService from '#services/StagingService'
+import IngestService, { segmentFromHeader } from '#services/IngestService'
+
+const segment = (name: string, k: number, n: number, id = `${name}-${k}@x`) =>
+  segmentFromHeader(
+    { 'subject': `"${name}" yEnc (${k}/${n})`, 'from': 'poster@x', 'message-id': `<${id}>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
+    'alt.binaries.test',
+    k
+  )!
+
+async function binary(name: string) {
+  return db.from('binaries').where('name', name).first()
+}
+
+test.group('Staging', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('a lone segment waits; its second segment brings both in', async ({ assert }) => {
+    let r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
+    assert.equal(r.staged, 1)
+    assert.lengthOf(r.ingest, 0)
+    await IngestService.ingest(r.ingest)
+    assert.isNull(await binary('rnd1.bin'))
+
+    // The same article again (a re-fetched range) is no second segment.
+    r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
+    assert.lengthOf(r.ingest, 0)
+
+    // Nor is the same article crossposted to another group (other group, other article number).
+    const crosspost = { ...segment('rnd1.bin', 3, 50), groups: ['alt.binaries.boneless'], articleNumber: 160154327571 }
+    r = await StagingService.filter([crosspost])
+    assert.lengthOf(r.ingest, 0)
+    r = await StagingService.filter([segment('dup.bin', 2, 9), { ...segment('dup.bin', 2, 9), groups: ['alt.binaries.boneless'] }])
+    assert.equal(r.staged, 1)
+
+    r = await StagingService.filter([segment('rnd1.bin', 4, 50)])
+    assert.deepEqual(r.ingest.map((s) => s.number).sort(), [3, 4])
+    await IngestService.ingest(r.ingest)
+    const b = await binary('rnd1.bin')
+    assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 2)
+    assert.lengthOf(await db.from('staged_segments').where('file_key', segment('rnd1.bin', 3, 50).fileKey), 0)
+
+    // Later segments of a known file go straight in.
+    r = await StagingService.filter([segment('rnd1.bin', 9, 50)])
+    assert.lengthOf(r.ingest, 1)
+  })
+
+  test('single-part files and files with two segments in the batch go straight in; staged ones join them', async ({ assert }) => {
+    // Part 1 arrived alone at the end of the previous batch.
+    await StagingService.filter([segment('real.part01.rar', 1, 3)])
+    const r = await StagingService.filter([segment('real.part01.rar', 2, 3), segment('real.part01.rar', 3, 3), segment('one.nfo', 1, 1)])
+    assert.equal(r.staged, 0)
+    assert.sameMembers(r.ingest.map((s) => `${s.name}:${s.number}`), ['real.part01.rar:1', 'real.part01.rar:2', 'real.part01.rar:3', 'one.nfo:1'])
+  })
+
+  test('GC adopts segments whose file got a binary, and drops old ones', async ({ assert }) => {
+    await StagingService.filter([segment('late.bin', 7, 10)])
+    await StagingService.filter([segment('old.bin', 2, 10)])
+    await db.from('staged_segments').where('file_key', segment('old.bin', 2, 10).fileKey).update({ staged_at: new Date(Date.now() - 3 * 3600_000) })
+    // Another job created the binary meanwhile.
+    await IngestService.ingest([segment('late.bin', 1, 10), segment('late.bin', 2, 10)])
+    const result = await StagingService.sweep(2 * 3600, Date.now() + 10_000)
+    assert.deepEqual(result, { adopted: 1, expired: 1 })
+    const b = await binary('late.bin')
+    assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 3)
+  })
+})

tests/functional/web_api.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
 import testUtils from '@adonisjs/core/services/test_utils'
 
 async function seed() {
@@ -8,12 +9,12 @@ async function seed() {
     .table('releases')
     .insert({ release_key: 'w'.repeat(40), name: 'Webtest.Show.S01E02.1080p', source: 'par2', groups: '["alt.binaries.test"]', posted_at: now, first_seen_at: now, last_seen_at: now, contents: JSON.stringify([{ name: 'Webtest.Show.S01E02.1080p/episode.mkv', size: 1000 }]) })
     .returning('id')
-  await db.table('files').insert({
+  await FileStore.insert([{
     filename: 'Xq8r2KdLm3Zw91PqA7webtest', real_name: 'Webtest.Show.S01E02.1080p.mkv', poster: 'p@example.invalid',
     date: now.getTime(), parts: 2, size: 1400, release_id: release.id, groups: '["alt.binaries.test"]',
-    message_ids: JSON.stringify({ '1': { id: 'one@test', size: 700 }, '2': { id: 'two@test', size: 700 } }),
+    segments: [{ number: 1, id: 'one@test', size: 700 }, { number: 2, id: 'two@test', size: 700 }],
     created_at: now, updated_at: now,
-  })
+  }])
   return Number(release.id)
 }
 
@@ -67,21 +68,21 @@ test.group('Web API', (group) => {
     const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete
     const file = (releaseId: number, name: string) => ({
       filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
-      groups: '[]', message_ids: '{}', created_at: now, updated_at: now,
+      groups: '[]', segments: [], created_at: now, updated_at: now,
     })
 
     // Counter rule: [i/M] announced 2 files.
     const [counted] = await db.table('releases').insert({ release_key: 'c'.repeat(40), name: 'Counted', groups: '[]', file_total: 2, first_seen_at: now, last_seen_at: now }).returning('id')
-    await db.table('files').insert(file(counted.id, 'a.rar'))
+    await FileStore.insert([file(counted.id, 'a.rar')])
     assert.isFalse(await complete(counted.id))
-    await db.table('files').insert(file(counted.id, 'b.rar'))
+    await FileStore.insert([file(counted.id, 'b.rar')])
     assert.isTrue(await complete(counted.id))
 
     // par2 rule: the listed file becomes present once a file of its exact size is there.
     const [listed] = await db.table('releases').insert({ release_key: 'l'.repeat(40), name: 'Listed', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id')
     await db.table('release_files').insert({ release_id: listed.id, par2_file_id: '1'.repeat(32), name: 'Listed.mkv', size: 5_000_000, md5_16k: '0'.repeat(32) })
     assert.isFalse(await complete(listed.id))
-    const [f] = await db.table('files').insert(file(listed.id, 'Xq8r2KdLm3Zw91PqA7')).returning('id')
+    const [f] = await FileStore.insert([file(listed.id, 'Xq8r2KdLm3Zw91PqA7')])
     const [b] = await db.table('binaries').insert({ file_key: 'z'.repeat(40), poster: 'p', subject: '"x" yEnc', name: 'x', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, file_id: f.id }).returning('id')
     assert.isFalse(await complete(listed.id))
     await db.from('binaries').where('id', b.id).update({ size: 5_000_000 })
@@ -94,8 +95,8 @@ test.group('Web API', (group) => {
       .insert({ release_key: 'p'.repeat(40), name: 'Webtest.Par2only.S01E01.1080p.WEB-DL-GRP', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
     await db.table('release_files').insert({ release_id: release.id, par2_file_id: '2'.repeat(32), name: 'Webtest.Par2only.S01E01.mkv', size: 1_000_000_000, md5_16k: '0'.repeat(32) })
-    await db.table('files').insert({ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
-      release_id: release.id, groups: '[]', message_ids: '{}', created_at: now, updated_at: now })
+    await FileStore.insert([{ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
+      release_id: release.id, groups: '[]', segments: [], created_at: now, updated_at: now }])
 
     const hidden = await client.get('/api/v1/releases').qs({ q: 'webtest par2only' })
     assert.notInclude(hidden.body().releases.map((r: any) => r.id), Number(release.id))

tests/functional/yenc_name.spec.ts

@@ -0,0 +1,40 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import ContentService from '#services/ContentService'
+
+let keys = 0
+async function counterFile(name: string) {
+  const now = new Date()
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `yenc${++keys}`.padEnd(40, 'y'), name, source: 'weak-stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  const [{ id }] = await FileStore.insert([
+    { filename: name, poster: 'bob@home.com', date: now.getTime(), parts: 2, size: 10, release_id: r.id, groups: '["alt.binaries.boneless"]',
+      segments: [{ number: 1, id: `${name}@nyuu`, size: 10 }], created_at: now, updated_at: now },
+  ])
+  return db.from('files').where('id', id).first()
+}
+
+test.group('Names from the yEnc header', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('random-named files take the yEnc name and gather in its release', async ({ assert }) => {
+    const a = await counterFile('P9BSPl59iDfQUVvXFeBVG4MDQaOTufds')
+    const b = await counterFile('Faoq6nlEaVlTzYyNA27sSfFm2KvLHOvD')
+    const nameFromYenc = (ContentService as any).nameFromYenc.bind(ContentService)
+    assert.equal(await nameFromYenc(a, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar'), 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar')
+    await nameFromYenc(b, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part02.rar')
+    const [fa, fb] = await Promise.all([db.from('files').where('id', a.id).first(), db.from('files').where('id', b.id).first()])
+    assert.equal(fa.release_id, fb.release_id)
+    assert.equal((await db.from('releases').where('id', fa.release_id).first()).name, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP')
+
+    // A random yEnc name, or a file that has a real name already, stays as it is.
+    const c = await counterFile('Vg9QflfNLXVMPOWtK4KRJv474DadgTIF')
+    assert.isUndefined(await nameFromYenc(c, null, 'qTcrIl1qTo4Bx6El5c4RtTA4ZSqwd6tz'))
+    const d = await counterFile('Some.Release.part01.rar')
+    assert.isUndefined(await nameFromYenc(d, null, 'Other.Name.part01.rar'))
+  })
+})