Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/controllers/newznab_controller.ts

@@ -101,7 +101,9 @@ export default class NewznabController {
         postedAt: r.postedAt ? new Date(r.postedAt) : null,
         poster: r.poster,
         groups: r.groups ?? [],
-        password: Boolean(r.encrypted),
+        // 1 only when a password is needed that we can't give; a known one is in our NZB.
+        password: r.passwordState === 'missing',
+        description: r.spotDescription ?? undefined,
         season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
         episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
       }

app/controllers/releases_controller.ts

@@ -18,6 +18,8 @@ export default class ReleasesController {
       sort: qs.sort as SortOrder | undefined,
       page: Number(qs.page) || 1,
       perPage: Number(qs.per_page) || 50,
+      // The page shows "1,000+"; counting further made filter clicks take seconds.
+      countLimit: 1000,
     })
   }
 
@@ -37,6 +39,16 @@ export default class ReleasesController {
     return response.send(nzb.xml)
   }
 
+  /** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */
+  async image({ params, response }: HttpContext) {
+    const image = await SearchService.spotImage(Number(params.id))
+    if (!image) return response.notFound({ error: 'No image' })
+    response.header('Content-Type', image.type)
+    // A spot's image never changes.
+    response.header('Cache-Control', 'public, max-age=604800, immutable')
+    return response.send(image.data)
+  }
+
   /** GET /api/v1/stats */
   async stats() {
     return SearchService.stats()

app/models/file.ts

@@ -1,6 +1,8 @@
 import { DateTime } from 'luxon'
 import { column } from '@adonisjs/lucid/orm'
 import BaseModel from '#models/base_model'
+import { loadedSegments, storedSegments } from '#services/FileStore'
+import type { Segment } from '#lib/nzb/index'
 
 export default class File extends BaseModel {
   @column({ isPrimary: true })
@@ -21,8 +23,12 @@ export default class File extends BaseModel {
   @column()
   declare parts: number
 
-  @column({ columnName: 'message_ids' })
-  declare messageIds: Record<string, any>
+  /** Stored compressed (see FileStore); the model sees the list. */
+  @column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) })
+  declare segments: Segment[]
+
+  @column({ columnName: 'first_message_id' })
+  declare firstMessageId: string | null
 
   @column()
   declare groups: string[]

app/services/ArchiveService.ts

@@ -7,7 +7,7 @@ import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import IngestService from '#services/IngestService'
 import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem, rarReleaseIdentity } from '#lib/release/index'
+import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
 
 /** Stored per file and release; a listing of thousands of files is not useful. */
 const MAX_ENTRIES = 500
@@ -59,7 +59,9 @@ class ArchiveService {
       update.encrypted = true
     }
     const better = releaseNameFromContents(info)
-    const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
+    // A spot's title is kept (CONCEPT.md D29).
+    const renamed =
+      release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
     if (renamed) update.name = renamed
     if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
 
@@ -138,7 +140,7 @@ export async function renameFromStoredContents(): Promise<number> {
     .select('id', 'name', 'source', 'contents')
   let renamed = 0
   for (const release of releases) {
-    if (!looksRandom(release.name) && release.source !== 'post') continue
+    if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
     const better = releaseNameFromContents({ entries: release.contents })
     if (!better || better === release.name) continue
     await db.from('releases').where('id', release.id).update({ name: better })
@@ -147,11 +149,27 @@ export async function renameFromStoredContents(): Promise<number> {
   return renamed
 }
 
-/** Obfuscated names: one long token of mixed letters and digits, no separators. */
+/**
+ * Obfuscated names: one long token of mixed letters and digits, no
+ * separators, and the other generated shapes seen on Usenet (LOG.md
+ * 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
+ */
 export function looksRandom(name: string): boolean {
   // An unknown short extension (e.g. ".bin") says nothing about the name.
   name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
+  // VIDEO_TS, Sample, CD1: real words, but no name.
+  if (isGenericName(name)) return true
   if (/^[0-9a-f]{16,}$/i.test(name)) return true
+  // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
+  if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
+  // UUIDs
+  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
+  // 1650777167 (yEncBin), "83111763 n" (rclone uploads)
+  if (/^\d{6,}( n)?$/.test(name)) return true
+  // prvjr7hapd0i1ny14pz4, and rclone's long base32 names
+  if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
+  // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
+  if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
   if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
   // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
   return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6

app/services/BackfillService.ts

@@ -0,0 +1,104 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import FetchRangeService from '#services/FetchRangeService'
+import { articleAtDate } from '#lib/nntp/date_search'
+
+/** Articles XOVERed to find a posting date near an article number. */
+const PROBE = 200n
+
+/**
+ * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
+ * where live indexing began back to the article posted N days ago; `tick`
+ * (every scheduler run) queues low-priority, header-only fetch ranges from
+ * that cursor downwards, newest first, while few are waiting and the
+ * database stays within its size limits.
+ */
+class BackfillService {
+  /** Sets the group's backfill to reach back `days` days. Returns its range. */
+  async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
+    const group = await db.from('groups').where('name', groupName).first()
+    if (!group) throw new Error(`Unknown group ${groupName}`)
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
+      const dateOf = async (n: bigint) => {
+        const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
+        const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
+        const date = header ? Date.parse(header.date) : Number.NaN
+        return Number.isNaN(date) ? null : date
+      }
+      const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
+      // Live indexing began a little before the group was added: continue from there.
+      const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
+      const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
+      await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
+      return next > until ? { from: until, to: next } : null
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+
+  /** Stops a group's backfill (queued ranges still run). */
+  async stop(groupName: string) {
+    await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
+  }
+
+  /** Queues the next backfill ranges. Returns how many. */
+  async tick(): Promise<number> {
+    const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
+    const groups = await db
+      .from('groups')
+      .where('active', true)
+      .whereNotNull('backfill_until_id')
+      .whereRaw('backfill_next_id > backfill_until_id')
+      .orderBy('id')
+    if (groups.length === 0) return 0
+
+    const room = await this.room(maxDefaultGb, maxHddGb)
+    if (room !== true) {
+      logger.warn(`Backfill paused: ${room}.`)
+      return 0
+    }
+    const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
+    let free = maxQueued - Number(count)
+    let queued = 0
+    // Round-robin over the groups, one range each per pass.
+    while (free > 0 && groups.length > 0) {
+      for (const group of [...groups]) {
+        if (free <= 0) break
+        const next = BigInt(group.backfill_next_id)
+        const until = BigInt(group.backfill_until_id)
+        const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
+        await FetchRangeService.queue(group, start, next, { backfill: true })
+        group.backfill_next_id = (start - 1n).toString()
+        await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
+        free--
+        queued++
+        if (start - 1n <= until) {
+          groups.splice(groups.indexOf(group), 1)
+          logger.info(`${group.name}: backfill fully queued.`)
+        }
+      }
+    }
+    return queued
+  }
+
+  /** True when both parts of the database are under their limits, else why not. */
+  private async room(maxDefaultGb: number, maxHddGb: number): Promise<true | string> {
+    const { rows } = await db.rawQuery(
+      `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
+    )
+    const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
+    if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
+    if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
+    return true
+  }
+}
+
+export default new BackfillService()

app/services/ContentService.ts

@@ -6,10 +6,25 @@ import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#
 import NameService from '#services/NameService'
 import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
 import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
-import { sniffContentType, type ContentType } from '#lib/content/index'
+import {
+  isArchiveStart,
+  NICOVIDEO_FILE,
+  NICOVIDEO_RELEASE,
+  nicovideoInfo,
+  nicovideoReleaseName,
+  sniffContentType,
+  zstdPrefix,
+  type ContentType,
+  type NicovideoInfo,
+} from '#lib/content/index'
 import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
-import { fileStem } from '#lib/release/index'
+import { fileStem, releaseIdentity } from '#lib/release/index'
+import IngestService from '#services/IngestService'
 import { looksLikeFilename } from '#lib/subject/index'
+import { sevenZipVolume } from '#lib/sevenzip/index'
+import SevenZipService from '#services/SevenZipService'
+import SfvService from '#services/SfvService'
+import { segmentsOf } from '#services/FileStore'
 
 const FIRST_CHUNK = 16 * 1024
 
@@ -35,6 +50,12 @@ export interface ContentResult {
   nzb?: NzbImportResult
   /** NZBs read from inside an archive. */
   archiveNzbs?: NzbImportResult[]
+  /** Files an SFV's listing moved into its release. */
+  sfvMoved?: number
+  /** A niconico capture's release name from its API response (D34). */
+  nicovideo?: string
+  /** The real name from the yEnc header, for a file posted under a random subject (D36). */
+  yencName?: string
 }
 
 /**
@@ -48,14 +69,65 @@ class ContentService {
   /** Random or missing names, and named first RAR volumes (for contents and password flag). */
   isCandidate(name: string) {
     if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
+    if (isArchiveStart(name)) return true
     if (/\.nzb$/i.test(name)) return true
+    // SFVs name the files of their release; niconico captures carry their title (D34).
+    if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
+    // Every 7z volume: the set is listed when its last volume is in (see inspect).
+    if (sevenZipVolume(name) !== null) return true
     return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
   }
 
+  /**
+   * Uploads that randomise the subject per article often keep the real file
+   * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
+   * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
+   * leaves its weak one-file release for the release its name gives, where
+   * the other files of the upload gather. Returns the name, or undefined.
+   */
+  private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
+    const current = String(file.real_name ?? file.filename)
+    if (file.real_name || !looksRandom(fileStem(current))) return undefined
+    if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
+
+    const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
+    const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
+    let releaseId = file.release_id
+    if (weak) {
+      const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
+      if (identity) {
+        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
+          { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
+        ])
+        releaseId = ids.get(identity.key) ?? releaseId
+      }
+    }
+    await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
+    if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
+    return yencName
+  }
+
+  /**
+   * Names a niconico capture's release after the video, while it still has
+   * the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
+   * the search text.
+   */
+  private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
+    if (!file.release_id) return undefined
+    const release = await db.from('releases').where('id', file.release_id).first()
+    if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
+    const name = nicovideoReleaseName(info, id)
+    await db
+      .from('releases')
+      .where('id', release.id)
+      .update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
+    return name
+  }
+
   /** Downloads a whole (small) file whose first segment is already decoded. */
-  private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
-    const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
-    const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
+  private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
+    const parts = segmentsOf(file)
+    const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
     if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
     if (parts.length === 1) return first
 
@@ -63,7 +135,7 @@ class ContentService {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      for (const [, part] of parts) {
+      for (const part of parts) {
         const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
         assembled ??= YencFile.fromSegment(segment)
         assembled.add(segment)
@@ -83,7 +155,7 @@ class ContentService {
    * uploads by other posters.
    */
   private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
-    const archive = await this.readWhole(first, file.message_ids)
+    const archive = await this.readWhole(first, file)
     const info = archive && parseRarHeaders(archive)
     if (!archive || !info) return []
     const entries = info.entries.filter(isNzbEntry)
@@ -104,7 +176,13 @@ class ContentService {
     const file = await db.from('files').where('id', fileId).first()
     const binary = await db.from('binaries').where('file_id', fileId).first()
     if (!file || !this.isCandidate(file.filename)) return null
-    const first = file.message_ids?.['1']
+    // A later 7z volume: list the set from its first volume once all volumes are in.
+    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
+      const volumes = await SevenZipService.volumes(file)
+      if (!volumes || volumes[0].archive) return null
+      return this.inspect(Number(volumes[0].id))
+    }
+    const first = segmentsOf(file)[0]
     if (!first?.id) return null
 
     const conn = await NntpService.acquire()
@@ -142,8 +220,9 @@ class ContentService {
     }
 
     const result: ContentResult = { named: 0, contentType }
-    if (contentType === 'nzbe' || contentType === 'rclone') {
-      // An encrypted NZB, or rclone crypt (names and data encrypted): nothing
+    result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
+    if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
+      // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
       // to read, and useless without the key.
       const releaseId = file.release_id ?? binary?.release_id
       if (releaseId) {
@@ -151,14 +230,28 @@ class ContentService {
       }
       return result
     }
+    const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
+    if (nicovideo) {
+      const info = nicovideoInfo(await zstdPrefix(segment.data))
+      if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
+      return result
+    }
+    if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
+      const text = await this.readWhole(segment.data, file)
+      if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
+      return result
+    }
     if (contentType === 'nzb') {
-      const xml = await this.readWhole(segment.data, file.message_ids)
+      const xml = await this.readWhole(segment.data, file)
       if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
       return result
     }
-    const set = await Par2Service.read(segment.data, file.message_ids)
+    const set = await Par2Service.read(segment.data, segmentsOf(file))
     if (set && binary) {
       result.par2 = await Par2Service.attach(binary, set)
+    } else if (contentType === '7z') {
+      const info = await SevenZipService.list(file, segment.data)
+      if (info) result.archive = await ArchiveService.record(file, binary, info)
     } else {
       const info = parseRarHeaders(segment.data)
       if (info) {

app/services/FetchRangeService.ts

@@ -1,6 +1,7 @@
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
 
 /** A failed range is retried after this long... */
 const REQUEUE_AFTER_MS = 5 * 60 * 1000
@@ -20,21 +21,22 @@ interface Group {
  * retried instead of silently skipped. See CONCEPT.md (D18).
  */
 class FetchRangeService {
-  async queue(group: Group, startId: bigint, endId: bigint) {
+  /** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
+  async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
     const now = new Date()
     const [row] = await db
       .table('fetch_ranges')
-      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now })
+      .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
       .returning(['id', 'requeues'])
-    await this.addJob(group.name, Number(row.id), 0, startId, endId)
+    await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
   }
 
-  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) {
+  private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
     await QueueService.nntpFetchQueue.add(
       'fetch-headers',
-      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId },
+      { groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
       // The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
-      { jobId: this.jobId(rangeId, requeues) }
+      { jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
     )
   }
 
@@ -75,7 +77,7 @@ class FetchRangeService {
         .from('fetch_ranges')
         .where('id', range.id)
         .update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
-      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id))
+      await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
       requeued++
       logger.warn(
         `Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
@@ -85,13 +87,14 @@ class FetchRangeService {
   }
 
   /**
-   * Moves `groups.last_indexed_id` up to just below the first range that is
-   * not done yet, and forgets done ranges. Returns the new watermark.
+   * Moves `groups.last_indexed_id` up to just below the first live range
+   * that is not done yet (backfill ranges lie below it and don't count), and
+   * forgets done ranges. Returns the new watermark.
    */
   async advanceWatermark(group: Group): Promise<bigint> {
     const { rows } = await db.rawQuery(
       `UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
-         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'),
+         (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
          g.last_queued_id))
        WHERE g.id = ? RETURNING last_indexed_id`,
       [group.id]

app/services/FileStore.ts

@@ -0,0 +1,55 @@
+import db from '@adonisjs/lucid/services/db'
+import type { QueryClientContract } from '@adonisjs/lucid/types/database'
+import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
+
+/**
+ * The only code that knows how `files` keeps its segment lists (CONCEPT.md
+ * D31): compressed in `files.segments`, with the first segment's message-id
+ * in the indexed `first_message_id` for lookups. Read them with
+ * `segmentsOf(file)`; write new files with `FileStore.insert`.
+ */
+
+/** A new `files` row, with its segments as a list. */
+export type NewFile = Record<string, unknown> & { segments: Segment[] }
+
+const unpacked = new WeakMap<object, Segment[]>()
+
+/** The segments of a `files` row (a row from `db.from('files')`), in order. */
+export function segmentsOf(file: { segments?: Buffer | Segment[] | null }): Segment[] {
+  const { segments } = file
+  if (!segments) return []
+  if (Array.isArray(segments)) return segments
+  let list = unpacked.get(file)
+  if (!list) {
+    list = unpackSegments(segments)
+    unpacked.set(file, list)
+  }
+  return list
+}
+
+/** Stored form of a segment list, for the File model's column. */
+export function storedSegments(segments: Segment[]) {
+  return packSegments(segments)
+}
+
+export function loadedSegments(stored: Buffer | null): Segment[] {
+  return stored ? unpackSegments(stored) : []
+}
+
+class FileStore {
+  /**
+   * Inserts files, skipping those whose `file_key` exists. Returns the rows
+   * that were inserted (id and filename).
+   */
+  async insert(files: NewFile[], client: Pick<QueryClientContract, 'knexQuery'> = db): Promise<{ id: number; filename: string }[]> {
+    if (files.length === 0) return []
+    const rows = files.map(({ segments, ...file }) => {
+      const sorted = [...segments].sort((a, b) => a.number - b.number)
+      return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
+    })
+    const inserted = await client.knexQuery().table('files').insert(rows).onConflict('file_key').ignore().returning(['id', 'filename'])
+    return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
+  }
+}
+
+export default new FileStore()

app/services/IndexMaintenance.ts

@@ -0,0 +1,85 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/** Only indexes at least this big are worth checking. */
+const MIN_BYTES = 50 * 1024 * 1024
+/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
+const MAX_LEAF_DENSITY = 50
+
+export interface Rebuilt {
+  index: string
+  density: number
+  before: number
+  after: number
+}
+
+/**
+ * GC deletes millions of rows (single-article binaries, copied parts, empty
+ * releases). Vacuum makes their index entries reusable but doesn't compact
+ * the index, so B-trees on the churning tables end up 10–20% full (LOG.md
+ * 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
+ * D33). Needs the pgstattuple extension.
+ */
+class IndexMaintenance {
+  async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
+    const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
+    if (extension.length === 0) {
+      logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
+      return []
+    }
+    if (!dryRun) await this.dropLeftovers()
+
+    const { rows: candidates } = await db.rawQuery(
+      `SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
+       FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
+       JOIN pg_namespace n ON n.oid = c.relnamespace
+       WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
+       ORDER BY 2 DESC`,
+      [MIN_BYTES]
+    )
+    const rebuilt: Rebuilt[] = []
+    for (const { index, bytes } of candidates) {
+      const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
+      const density = Number(rows[0].avg_leaf_density)
+      if (density >= MAX_LEAF_DENSITY) continue
+      if (!dryRun && !(await this.reindex(index))) continue
+      const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
+      const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
+      rebuilt.push(result)
+      logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
+    }
+    return rebuilt
+  }
+
+  /**
+   * REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
+   * binaries during a backfill). Its half-built copy is dropped and it is
+   * tried once more; a second failure skips the index until the next run.
+   */
+  /** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
+  private async dropLeftovers(index = '') {
+    const { rows } = await db.rawQuery(
+      `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
+       WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
+      [index]
+    )
+    for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
+  }
+
+  private async reindex(index: string): Promise<boolean> {
+    for (let attempt = 1; attempt <= 2; attempt++) {
+      try {
+        await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
+        return true
+      } catch (error: any) {
+        logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
+        await this.dropLeftovers(index)
+      }
+    }
+    return false
+  }
+}
+
+const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
+
+export default new IndexMaintenance()

app/services/NameService.ts

@@ -112,7 +112,7 @@ export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; r
   const { par2ReleaseName } = await import('#lib/release/index')
   const listings = await db.rawQuery(
     `SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
-     JOIN release_files rf ON rf.release_id = r.id GROUP BY r.id, r.name`
+     JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
   )
   let renamed = 0
   for (const release of listings.rows) {

app/services/NntpService.ts

@@ -1,5 +1,5 @@
 import nntpConfig from '#config/nntp'
-import { NNTP } from "nntp-js";
+import { NNTP } from "#lib/nntp/nntp_js";
 import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
 
 class NntpService {
@@ -94,6 +94,20 @@ class NntpService {
     return parseGroupResponse(resp)
   }
 
+  /**
+   * Rejects when `promise` hasn't settled within `ms`. nntp-js has no
+   * timeouts: a connection the server stops answering (still ESTABLISHED,
+   * nothing in flight) leaves its command waiting forever. The caller must
+   * then release the connection as failed, so it's destroyed.
+   */
+  public timed<T>(promise: Promise<T>, what: string, ms = nntpConfig.commandTimeoutMs): Promise<T> {
+    let timer: NodeJS.Timeout
+    const timeout = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`NNTP ${what}: no answer within ${ms / 1000} s`)), ms)
+    })
+    return Promise.race([promise, timeout]).finally(() => clearTimeout(timer))
+  }
+
   /** Fetches a raw (still dot-stuffed) article body as a Buffer. */
   public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
     const id = messageId.startsWith('<') ? messageId : `<${messageId}>`

app/services/NzbImportService.ts

@@ -7,6 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
 import { parseSubject } from '#lib/subject/index'
 import { fileStem } from '#lib/release/index'
 import { looksRandom } from '#services/ArchiveService'
+import QueueService from '#services/QueueService'
+import FileStore from '#services/FileStore'
+import { isArchiveStart } from '#lib/content/index'
 
 export interface NzbImportResult {
   releaseId: number
@@ -20,8 +23,29 @@ export interface NzbImportResult {
 /** Filename of an NZB entry, from its subject (tolerating odd quoting). */
 export function nzbFileName(subject: string): string {
   const parsed = parseSubject(subject)
-  const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
-  return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
+  const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
+  if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
+  return unquotedFileName(subject)
+}
+
+/**
+ * A filename from a subject without quotes, as some spot NZBs have:
+ * `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
+ * `06 - Song.flac  Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
+ * trailing size are dropped; then the name ends at the first extension
+ * followed by a space.
+ */
+export function unquotedFileName(subject: string): string {
+  const cleaned = subject
+    .replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
+    .replace(/\s\d+\s*$/, ' ')
+    .replace(/(\s+-)+\s*$/, '')
+    .replace(/^\s*-\s+/, '')
+    .replace(/\s+/g, ' ')
+    .trim()
+  const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
+  const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
+  return name.replace(/\s+-$/, '').trim() || subject.trim()
 }
 
 export interface NzbImportOptions {
@@ -36,6 +60,13 @@ export interface NzbImportOptions {
   name?: string
   /** Release source for new releases (default `nzb`). */
   source?: ReleaseIdentity['source']
+  /**
+   * The NZB is authoritative for which files form the release and what it is
+   * called (Spotnet spots): its files get the NZB's own release, named
+   * `name`, also when name rules had put them in another release. Files
+   * another trusted NZB claimed first stay where they are.
+   */
+  trusted?: boolean
 }
 
 /**
@@ -55,21 +86,39 @@ class NzbImportService {
       firstId: f.segments[0].id,
     }))
 
-    return db.transaction(async (trx) => {
+    const added: { id: number; filename: string }[] = []
+    const result = await db.transaction(async (trx) => {
       const known = await trx
         .from('files')
-        .whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
-        .select('id', 'release_id', `message_ids`)
-      const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
+        .whereIn('first_message_id', entries.map((e) => e.firstId))
+        .select('id', 'release_id', 'first_message_id')
+      const knownIds = new Set(known.map((f) => f.first_message_id))
 
       // Join the release the listed files are already in, else the posted NZB's own, else a new one.
       const counts = new Map<number, number>()
       for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
       let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
       const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
+      const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
+      const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
+
+      // Releases of other trusted NZBs: their files are not taken over.
+      let claimed = new Set<number>()
+      if (options.trusted) {
+        // Only while that NZB still stands: a disposed spot's files go to its correction.
+        const holders = await trx
+          .from('releases as r')
+          .whereIn('r.id', [...counts.keys()])
+          .where('r.source', 'spot')
+          .whereNot('r.release_key', key)
+          .whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
+          .select('r.id')
+        claimed = new Set(holders.map((r) => Number(r.id)))
+        // Join another spot's release when it holds most of these files (a repost); else use our own.
+        if (releaseId && !claimed.has(releaseId)) releaseId = undefined
+        if (releaseId) claimed.delete(releaseId)
+      }
       if (!releaseId) {
-        const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
-        const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
         const ids = await IngestService.upsertReleases(trx.knexClient, [
           {
             release: { key, name: title, source: options.source ?? 'nzb' },
@@ -84,9 +133,8 @@ class NzbImportService {
       const now = new Date()
       const missing = entries.filter((e) => !knownIds.has(e.firstId))
       for (let i = 0; i < missing.length; i += 500) {
-        await trx
-          .knexClient('files')
-          .insert(
+        added.push(
+          ...(await FileStore.insert(
             missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
               file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
               filename: name,
@@ -94,22 +142,23 @@ class NzbImportService {
               date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
               parts: file.segments.length,
               size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
-              message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
+              segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
               groups: JSON.stringify(file.groups),
               release_id: releaseId,
               created_at: now,
               updated_at: now,
-            }))
-          )
-          .onConflict('file_key')
-          .ignore()
+            })),
+            trx
+          ))
+        )
       }
       // The poster's NZB says which files form the post: known files join this
       // release, also when name rules had split them over several.
-      const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
+      const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
+      const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
       await trx
         .from('files')
-        .whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
+        .whereIn('id', moving.map((f) => f.id))
         .update({ release_id: releaseId })
       if (previous.length) {
         await trx.rawQuery(
@@ -130,9 +179,15 @@ class NzbImportService {
 
       const release = await trx.from('releases').where('id', releaseId).first()
       const update: Record<string, unknown> = {}
-      if (nzb.meta.password) update.password = nzb.meta.password
+      if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
       if (!release.file_total) update.file_total = entries.length
-      if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
+      if (options.trusted) {
+        // Only its own release: a repost joining another spot's release keeps that one's title.
+        if (release.release_key === key && release.name !== title) update.name = title
+        if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
+      } else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
+        update.name = title
+      }
       if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
 
       const result = {
@@ -149,6 +204,14 @@ class NzbImportService {
       )
       return result
     })
+    // Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
+    const archives = added.filter((f) => isArchiveStart(f.filename))
+    if (archives.length) {
+      await QueueService.collectionQueue.addBulk(
+        archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
+      )
+    }
+    return result
   }
 
   /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */

app/services/Par2Service.ts

@@ -1,4 +1,5 @@
 import db from '@adonisjs/lucid/services/db'
+import type { Segment } from '#lib/nzb/index'
 import logger from '@adonisjs/core/services/logger'
 import NntpService from '#services/NntpService'
 import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
@@ -35,11 +36,9 @@ class Par2Service {
    * Reads a par2 file whose first segment is already decoded, downloading the
    * remaining parts if there are any. Returns null when it is not par2.
    */
-  async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
+  async read(first: Buffer, segments: Segment[]): Promise<Par2Set | null> {
     if (!isPar2(first)) return null
-    const ids = Object.entries(messageIds)
-      .sort(([a], [b]) => Number(a) - Number(b))
-      .map(([, part]) => part.id)
+    const ids = segments.map((part) => part.id)
     if (ids.length > MAX_PARTS) return null
     if (ids.length === 1) return parsePar2(first)
 
@@ -143,9 +142,11 @@ class Par2Service {
       )
 
       // The set may have joined a release named after one random filename.
-      let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
+      const release = await trx.from('releases').where('id', releaseId).first()
+      let releaseName = release?.name
       const better = par2ReleaseName(set.files.map((f) => f.name))
-      if (releaseName && better && isBetterName(releaseName, better)) {
+      // A spot's title is kept (CONCEPT.md D29).
+      if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) {
         await trx.from('releases').where('id', releaseId).update({ name: better })
         releaseName = better
       }

app/services/ReleaseMergeService.ts

@@ -0,0 +1,42 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+
+/**
+ * Uploads split per poster (CONCEPT.md D35). Some uploaders post every
+ * volume of a RAR set under a different poster; the post-id grouping then
+ * makes one release per volume, all with the same name
+ * (`Masters.of.the.Universe.1987…`: 191 one-file releases of one 199-volume
+ * set). Such releases are one upload when all their files share one stem and
+ * no file name repeats (a repost would repeat them, another upload would
+ * bring another stem); their files move into the oldest of them, and GC
+ * deletes the emptied ones.
+ */
+class ReleaseMergeService {
+  async mergeSplitPosts({ dryRun = false } = {}): Promise<{ names: number; releases: number }> {
+    const { rows } = await db.rawQuery(`
+      WITH dup AS (
+        SELECT name FROM releases
+        WHERE file_count > 0 AND source = 'post' AND is_readable_name(name)
+        GROUP BY name HAVING count(*) > 1
+      )
+      SELECT r.name, array_agg(DISTINCT r.id ORDER BY r.id) AS ids
+      FROM dup JOIN releases r ON r.name = dup.name AND r.file_count > 0 AND r.source = 'post'
+      JOIN files f ON f.release_id = r.id
+      GROUP BY r.name
+      HAVING count(DISTINCT name_stem(coalesce(f.real_name, f.filename))) = 1
+        AND count(f.id) = count(DISTINCT coalesce(f.real_name, f.filename))
+        AND max(r.posted_at) - min(r.posted_at) < interval '30 days'`)
+    let releases = 0
+    for (const { name, ids } of rows as { name: string; ids: (string | number)[] }[]) {
+      const [target, ...others] = ids.map(Number)
+      releases += others.length
+      if (dryRun) continue
+      await db.from('files').whereIn('release_id', others).update({ release_id: target })
+      await db.from('binaries').whereIn('release_id', others).update({ release_id: target })
+      logger.info(`Merged ${others.length + 1} releases of one upload split per poster into ${target} "${name}".`)
+    }
+    return { names: rows.length, releases }
+  }
+}
+
+export default new ReleaseMergeService()

app/services/SearchService.ts

@@ -1,6 +1,20 @@
 import db from '@adonisjs/lucid/services/db'
 import { buildNzb } from '#lib/nzb/index'
 import SpotService from '#services/SpotService'
+import { segmentsOf } from '#services/FileStore'
+import { spotCategoryLabels } from '#lib/spotnet/index'
+
+/**
+ * `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
+ * without asking); `missing`: the archive is encrypted and we don't have
+ * it; `none`: no password needed as far as we know.
+ */
+export type PasswordState = 'known' | 'missing' | 'none'
+
+export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
+  if (hasPassword) return 'known'
+  return encrypted ? 'missing' : 'none'
+}
 
 export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
 
@@ -10,7 +24,7 @@ export interface SearchParams {
   readable?: boolean
   /** Only releases that have all their files. */
   complete?: boolean
-  /** Hide password-protected releases. */
+  /** Hide releases that need a password we don't have. */
   hideEncrypted?: boolean
   source?: string
   poster?: string
@@ -31,6 +45,8 @@ export interface SearchParams {
   hideContainers?: boolean
   /** Include incomplete releases that consist of par2 files only (hidden by default). */
   includePar2Only?: boolean
+  /** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
+  countLimit?: number
 }
 
 const SORTS: Record<SortOrder, string> = {
@@ -58,6 +74,24 @@ function pattern(word: string): string {
   return `%${escaped.replace(/[._\- ]+/g, '_')}%`
 }
 
+/**
+ * As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
+ * spaces as one space. `The.Block.AU` and `the block au` search the same.
+ */
+export function searchNormalize(text: string): string {
+  return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
+}
+
+/** How long the web UI's totals are reused. */
+const STATS_TTL_MS = 60 * 1000
+
+/**
+ * Counting stops here: a broad search needn't count every match. Each
+ * counted row is a random heap read, so the web UI counts less (it only
+ * shows "1,000+"); Newznab clients page by this total and keep the default.
+ */
+const COUNT_LIMIT = 10_000
+
 /**
  * Searching and browsing releases for the web interface (CONCEPT.md D22).
  */
@@ -66,21 +100,19 @@ class SearchService {
     const where: string[] = [
       VISIBLE_SQL,
       // Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
-      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`,
+      `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
     ]
     const bindings: unknown[] = []
 
-    const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8)
+    // Every word must occur in the release's search text: its name, archive
+    // listing and file names (CONCEPT.md D33), one trigram index for all.
+    const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
     for (const word of words) {
-      const p = pattern(word)
-      // Every word must match the release name, a file name, or the archive contents.
-      // One trigram-indexed lookup per table, not a subquery per release.
-      where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ?
-        OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`)
-      bindings.push(p, p, p, p)
+      where.push(`r.search_text LIKE ?`)
+      bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
     }
     if (params.readable) where.push('is_readable_name(r.name)')
-    if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE')
+    if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
     if (params.hideContainers) where.push('r.container IS NULL')
     // par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
     if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
@@ -121,6 +153,7 @@ class SearchService {
     const offset = params.offset ?? (page - 1) * perPage
     const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
     const whereSql = where.join(' AND ')
+    const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
 
     const [{ rows }, { rows: countRows }] = await Promise.all([
       db.rawQuery(
@@ -131,19 +164,22 @@ class SearchService {
                    SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
                    WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
                 jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
-                r.complete, ${PAR2_ONLY_SQL} AS par2_only
+                r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
+                (SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
          FROM releases r WHERE ${whereSql}
          ORDER BY ${order} LIMIT ? OFFSET ?`,
         [...bindings, perPage, offset]
       ),
-      db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings),
+      db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
     ])
 
     return {
       page,
       perPage,
       offset,
-      total: Number(countRows[0].total),
+      total: Math.min(Number(countRows[0].total), countLimit),
+      /** More than `total` match (the count stops at `countLimit`). */
+      totalCapped: Number(countRows[0].total) > countLimit,
       releases: rows.map((r: any) => ({
         id: Number(r.id),
         name: r.name,
@@ -163,6 +199,10 @@ class SearchService {
         category: r.category,
         container: r.container,
         par2Only: r.par2_only,
+        listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
+        hasPassword: r.has_password,
+        passwordState: passwordState(r.encrypted, r.has_password),
+        spotDescription: r.spot_description,
       })),
     }
   }
@@ -193,14 +233,32 @@ class SearchService {
       ),
     ])
 
-    const spot = release.source === 'spot'
-      ? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first()
-      : null
+    const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
+    const spot = spotRow ? await SpotService.withDetails(spotRow) : null
     return {
       id: Number(release.id),
       name: release.name,
       source: release.source,
-      spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null,
+      spot: spot
+        ? {
+            messageId: spot.message_id,
+            title: spot.title,
+            tag: spot.tag,
+            poster: spot.poster,
+            postedAt: spot.posted_at,
+            website: spot.website,
+            description: spot.description,
+            image: spot.image?.segments
+              ? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
+              : spot.image?.url
+                ? { external: spot.image.url }
+                : null,
+            categories: spotCategoryLabels(spot.category, spot.subcats),
+          }
+        : null,
+      password: release.password ?? null,
+      passwordSource: release.password_source ?? null,
+      passwordState: passwordState(release.encrypted, release.password !== null),
       listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
       poster: release.poster,
       groups: release.groups,
@@ -230,6 +288,22 @@ class SearchService {
     }
   }
 
+  /** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
+  async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
+    const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
+    if (!spot) return null
+    const data = await SpotService.image(spot)
+    if (!data) return null
+    const type =
+      data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
+      : data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
+      : data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
+      : data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
+      : data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
+      : null
+    return type ? { data, type } : null
+  }
+
   async nzb(id: number): Promise<{ name: string; xml: string } | null> {
     let release = await db.from('releases').where('id', id).first()
     if (!release) return null
@@ -244,39 +318,82 @@ class SearchService {
       .from('files')
       .where('release_id', id)
       .orderByRaw('coalesce(real_name, filename)')
-      .select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids')
+      .select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
     const xml = buildNzb(
       files.map((f) => ({
         name: f.real_name ?? f.filename,
         poster: f.poster,
         date: Number(f.date),
         groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
-        segments: f.message_ids,
+        segments: segmentsOf(f),
       })),
       { title: release.name, password: release.password ?? undefined }
     )
     return { name: release.name, xml }
   }
 
+  /**
+   * Totals for the web UI header. They count millions of rows (minutes with
+   * the backfill running), so they are computed at most once per
+   * STATS_TTL_MS and never twice at a time; callers in between share it.
+   */
   async stats() {
+    if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
+      this.statsRefresh ??= this.computeStats()
+        .then((value) => {
+          this.statsCache = { at: Date.now(), value }
+        })
+        // A failed background refresh keeps the previous totals; only a caller that waits sees the error.
+        .catch((error) => {
+          if (!this.statsCache) throw error
+        })
+        .finally(() => {
+          this.statsRefresh = undefined
+        })
+      // Only the very first caller waits; later ones get the previous totals meanwhile.
+      if (!this.statsCache) await this.statsRefresh
+    }
+    return this.statsCache!.value
+  }
+
+  private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
+  private statsRefresh?: Promise<void>
+
+  /**
+   * Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
+   * from Postgres's statistics (visible releases = the entries of an index
+   * that covers exactly them), ratios and sizes from a 0.5% sample of the
+   * releases table. Within a few percent once autovacuum has analysed.
+   */
+  private async computeStats() {
     const { rows } = await db.rawQuery(`
-      SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases,
-             (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases,
-             (SELECT count(*) FROM files) AS files,
-             (SELECT coalesce(sum(size), 0) FROM files) AS size,
-             (SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases,
+      WITH sample AS (
+        SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
+        FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
+      SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
+             (SELECT reltuples FROM pg_class WHERE relname = 'files') AS files,
+             (SELECT count(*) FROM sample) AS sampled,
+             (SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
+             -- Indexed data only: a spot listing's size is what the spot announces.
+             (SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
+             (SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
              (SELECT max(updated_at) FROM groups) AS last_indexed_at,
-             (SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC)
-                FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`)
+             (SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
+                FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
     const r = rows[0]
+    const releases = Math.max(0, Number(r.releases))
+    const sampled = Number(r.sampled)
+    // Sample counts scaled to all visible releases.
+    const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
     return {
-      releases: Number(r.releases),
-      readableReleases: Number(r.readable_releases),
-      files: Number(r.files),
-      size: Number(r.size),
-      encryptedReleases: Number(r.encrypted_releases),
+      approximate: true,
+      releases,
+      readableReleases: scale(r.sampled_readable),
+      files: Math.max(0, Number(r.files)),
+      size: scale(r.sampled_size),
+      encryptedReleases: scale(r.sampled_encrypted),
       lastIndexedAt: r.last_indexed_at,
-      sources: r.sources ?? [],
+      sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
     }
   }
 }

app/services/SevenZipService.ts

@@ -0,0 +1,202 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, open, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { promisify } from 'node:util'
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import NntpService from '#services/NntpService'
+import { segmentsOf } from '#services/FileStore'
+import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
+import type { RarInfo } from '#lib/rar/index'
+import {
+  checkHeader,
+  listingStart,
+  parseBsdtarListing,
+  parseSevenZipStart,
+  sevenZipVolume,
+  type SevenZipStart,
+} from '#lib/sevenzip/index'
+
+const run = promisify(execFile)
+
+/** The tail read for a listing: header plus packed header; more is not a header. */
+const MAX_TAIL_BYTES = 16 * 1024 * 1024
+
+/**
+ * Lists 7z archives, also split ones (.7z.001, .002, …), from the first
+ * segment of the first volume and the last segment(s) of the last one
+ * (CONCEPT.md D19). Those bytes are put at their place in a sparse file the
+ * size of the whole archive, which bsdtar (libarchive) lists: it only reads
+ * the start and the header at the end.
+ */
+class SevenZipService {
+  /**
+   * The volumes of the archive `file` belongs to, in order, when all of them
+   * up to the highest number seen are there; null otherwise. An unsplit (or
+   * unnamed) archive is its own only volume.
+   */
+  async volumes(file: any): Promise<any[] | null> {
+    const name = String(file.real_name ?? file.filename)
+    const volume = sevenZipVolume(name)
+    if (!volume) return [file]
+    if (!file.release_id) return null
+    const base = name.replace(/\.\d+$/, '').toLowerCase()
+    const rows = await db
+      .from('files')
+      .where('release_id', file.release_id)
+      .whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
+    const byNumber = new Map<number, any>()
+    for (const row of rows) {
+      const n = sevenZipVolume(String(row.real_name ?? row.filename))
+      if (n && !byNumber.has(n)) byNumber.set(n, row)
+    }
+    const max = Math.max(0, ...byNumber.keys())
+    const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
+    return max > 0 && ordered.every(Boolean) ? ordered : null
+  }
+
+  /**
+   * Lists the archive whose first volume starts with `firstData` (a decoded
+   * first segment). Null when it isn't a 7z start, when volumes are missing,
+   * or when the tail doesn't hold the header its start points to.
+   */
+  async list(firstFile: any, firstData: Buffer): Promise<RarInfo | null> {
+    const start = parseSevenZipStart(firstData)
+    if (!start) return null
+    const volumes = await this.volumes(firstFile)
+    if (!volumes) return null
+    if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1)
+
+    const tail = await this.readTail(volumes[volumes.length - 1], start)
+    if (!tail) return null
+
+    const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
+    try {
+      const archive = path.join(dir, 'archive.7z')
+      const handle = await open(archive, 'w')
+      try {
+        await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
+        await handle.write(tail.data, 0, tail.data.length, tail.from)
+        await handle.truncate(start.archiveSize)
+      } finally {
+        await handle.close()
+      }
+      return await this.bsdtar(archive, volumes.length > 1)
+    } finally {
+      await rm(dir, { recursive: true, force: true })
+    }
+  }
+
+  /** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
+  private async bsdtar(archive: string, split: boolean): Promise<RarInfo | null> {
+    let stdout = ''
+    try {
+      stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
+    } catch (error: any) {
+      if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split)
+      logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
+      return null
+    }
+    const entries = parseBsdtarListing(stdout)
+    const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
+    let encrypted = false
+    if (sample) {
+      try {
+        await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
+      } catch (error: any) {
+        // Without the data (it's all zeros here), anything else fails too: only this message counts.
+        encrypted = /content is encrypted/i.test(String(error.stderr))
+      }
+    }
+    return this.info(entries, false, encrypted, split)
+  }
+
+  private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo {
+    return {
+      format: '7z',
+      volume: split,
+      firstVolume: true,
+      solid: false,
+      headersEncrypted,
+      truncated: false,
+      entries: entries.map((e) => ({
+        name: e.name,
+        size: e.size,
+        packedSize: 0,
+        directory: e.directory,
+        encrypted: encrypted && !e.directory,
+        splitBefore: false,
+        splitAfter: false,
+        stored: false,
+        dataOffset: 0,
+      })),
+    }
+  }
+
+  /**
+   * The archive's tail from the last volume: the header (checked against its
+   * CRC) and, for an encoded header, the packed stream before it. `from` is
+   * its position in the whole archive.
+   */
+  private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> {
+    const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
+    const numbers = [...ids.keys()]
+    if (numbers.length === 0) return null
+    const segments = new Map<number, DecodedSegment>()
+    const segment = async (n: number) => {
+      if (!segments.has(n)) {
+        const id = ids.get(n)?.id
+        if (!id) throw new Error(`part ${n} of ${last.filename} is missing`)
+        segments.set(n, decodeSegment(await this.body(id), { lenientSize: true }))
+      }
+      return segments.get(n)!
+    }
+
+    const lastNumber = numbers[numbers.length - 1]
+    const lastSegment = await segment(lastNumber)
+    const volumeSize = lastSegment.meta.begin.size
+    const volumeStart = start.archiveSize - volumeSize
+    const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
+    const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
+    if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
+
+    // Bytes [from, to) of the whole archive, from the last volume's parts.
+    const read = async (from: number, to: number): Promise<Buffer | null> => {
+      if (from < volumeStart) return null
+      const a = from - volumeStart
+      const b = to - volumeStart
+      const pieces: Buffer[] = []
+      for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
+        const s = await segment(n)
+        const offset = (s.meta.part?.begin ?? 1) - 1
+        if (offset !== (n - 1) * partSize) return null
+        pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
+      }
+      const data = Buffer.concat(pieces)
+      return data.length === b - a ? data : null
+    }
+
+    const header = await read(start.headerStart, start.archiveSize)
+    if (!header || !checkHeader(start, header)) return null
+    const from = Math.min(listingStart(start, header), start.headerStart)
+    if (start.archiveSize - from > MAX_TAIL_BYTES) return null
+    const data = await read(from, start.archiveSize)
+    return data ? { from, data } : null
+  }
+
+  private async body(messageId: string): Promise<Buffer> {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
+    } catch (error: any) {
+      failed = !/^4\d\d/.test(String(error.message))
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+}
+
+export default new SevenZipService()

app/services/SfvService.ts

@@ -0,0 +1,80 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import { parseSfv, sfvReleaseName } from '#lib/content/index'
+
+/** Releases made by name rules alone; an SFV's listing outranks them. */
+const WEAK_SOURCES = ['stem', 'weak-stem', 'subject']
+/** Files further apart than this from the SFV in posting time aren't taken. */
+const WINDOW_MS = 2 * 24 * 60 * 60 * 1000
+
+/**
+ * SFV grouping (CONCEPT.md D34). An uploader that posts every track of an
+ * album as its own file (13.5k such releases from one poster on homelab)
+ * still posts the album's SFV, which names them all.
+ */
+class SfvService {
+  /**
+   * Records an SFV file's listing on its release, moves the listed files that
+   * sit in weak releases into it, and names the release after the album when
+   * it's still named after a track or the SFV itself. Returns how many files
+   * moved, or null when it isn't an SFV.
+   */
+  async attach(sfvFile: any, text: string): Promise<number | null> {
+    const entries = parseSfv(text)
+    if (entries.length === 0 || !sfvFile.release_id) return null
+    const releaseId = Number(sfvFile.release_id)
+    const postedAt = sfvFile.date ? new Date(Number(sfvFile.date)) : null
+    await db
+      .knexQuery()
+      .table('sfv_entries')
+      .insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
+      .onConflict(['release_id', 'name'])
+      .ignore()
+    await db.from('files').where('id', sfvFile.id).update({ content_type: 'sfv' })
+
+    const candidates = await db
+      .from('files as f')
+      .join('releases as r', 'r.id', 'f.release_id')
+      .whereIn(db.raw('coalesce(f.real_name, f.filename)') as any, entries.map((e) => e.name))
+      .whereNot('f.release_id', releaseId)
+      .whereIn('r.source', WEAK_SOURCES)
+      .select('f.id', 'f.poster', 'f.date')
+    const moving = candidates.filter((f) => this.sameUpload(f, sfvFile.poster, postedAt))
+    await this.move(moving.map((f) => Number(f.id)), releaseId)
+
+    const release = await db.from('releases').where('id', releaseId).first()
+    const album = sfvReleaseName(String(sfvFile.real_name ?? sfvFile.filename))
+    if (release && WEAK_SOURCES.includes(release.source) && album !== release.name && /^\d{1,3}-/.test(release.name)) {
+      await db.from('releases').where('id', releaseId).update({ name: album })
+    }
+    logger.info(`SFV "${sfvFile.filename}": ${entries.length} files listed, ${moving.length} moved into release ${releaseId} "${album}".`)
+    return moving.length
+  }
+
+  /** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
+  async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
+    const entries = await db.from('sfv_entries').where('name', file.name)
+    const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
+    if (!entry || Number(entry.release_id) === Number(file.release_id)) return null
+    if (file.release_id) {
+      const current = await db.from('releases').where('id', file.release_id).first()
+      if (current && !WEAK_SOURCES.includes(current.source)) return null
+    }
+    await this.move([file.id], Number(entry.release_id))
+    return Number(entry.release_id)
+  }
+
+  private sameUpload(file: { poster?: string | null; date?: number | string | null }, poster: string | null, postedAt: Date | null) {
+    if (poster && file.poster && file.poster !== poster) return false
+    if (postedAt && file.date && Math.abs(Number(file.date) - postedAt.getTime()) > WINDOW_MS) return false
+    return true
+  }
+
+  private async move(fileIds: number[], releaseId: number) {
+    if (fileIds.length === 0) return
+    await db.from('files').whereIn('id', fileIds).update({ release_id: releaseId })
+    await db.from('binaries').whereIn('file_id', fileIds).update({ release_id: releaseId })
+  }
+}
+
+export default new SfvService()

app/services/SpotService.ts

@@ -1,28 +1,55 @@
+import { createHash } from 'node:crypto'
 import db from '@adonisjs/lucid/services/db'
 import logger from '@adonisjs/core/services/logger'
 import indexerConfig from '#config/indexer'
 import NntpService from '#services/NntpService'
 import NzbImportService from '#services/NzbImportService'
-import { type OverviewHeader } from '#services/IngestService'
+import IngestService, { type OverviewHeader } from '#services/IngestService'
 import { parseNzb } from '#lib/nzb/index'
 import {
   moderationTarget,
+  passwordFromSpot,
   spotterId,
   decodeSpotBinary,
+  decodeSpotText,
   parseSpotArticleHeaders,
   parseSpotHeader,
   parseSpotXml,
   verifySpotHeader,
   type SpotHeader,
+  type SpotXml,
 } from '#lib/spotnet/index'
 
 /** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
 const MAX_NZB_SEGMENTS = 20
 /** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
 const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
+/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
+const INSERT_CHUNK = 2000
+/** Spot images larger than this many articles aren't fetched. */
+const MAX_IMAGE_SEGMENTS = 10
 /** A claimed spot whose worker died is taken again after this long. */
 const RECLAIM_AFTER = '1 hour'
 
+/**
+ * A spot's title: from its XML, which is decoded properly (the Subject of a
+ * raw header is read as ISO-8859-1), else from the header.
+ */
+function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
+  return head.xml?.title || head.header.title
+}
+
+/**
+ * A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
+ * spots): less than 1% of the size the spot announces (the real NZB is
+ * handed out elsewhere), or under 16 KB in all (text-only spots: requests,
+ * announcements). Its files are not imported, and the spot is hidden.
+ */
+export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
+  const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
+  return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
+}
+
 export function isSpotGroup(name: string): boolean {
   return indexerConfig.spotGroups.includes(name)
 }
@@ -55,74 +82,115 @@ class SpotService {
         posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
         key_id: spot.keyId,
         spotter_id: spotterId(spot.selfSignedPubKey) || null,
+        // From the overview; titles with accents may fail here (UTF-8 decoding)
+        // and are verified again on their raw header by worker:spots.
+        verified: verifySpotHeader(spot),
         created_at: now,
       })
     }
-    if (rows.length === 0) return 0
-    await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
+    for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
+      await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
+    }
     return rows.length
   }
 
-  /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
+  /**
+   * Takes up to `limit` pending spots, marked `processing`: recent ones
+   * (to be imported) first, then the history in posting order, so live spots
+   * don't wait behind a backfill and moderation follows its targets.
+   */
   async claim(limit: number): Promise<any[]> {
-    const { rows } = await db.rawQuery(
-      `UPDATE spots SET status = 'processing', processed_at = now()
-       WHERE message_id IN (
-         SELECT message_id FROM spots
-         WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
-         ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
-       RETURNING *`,
-      [limit]
-    )
+    const claim = async (filter: string, bindings: unknown[], n: number) =>
+      (
+        await db.rawQuery(
+          `UPDATE spots SET status = 'processing', processed_at = now()
+           WHERE message_id IN (
+             SELECT message_id FROM spots WHERE ${filter}
+             ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
+           RETURNING *`,
+          [...bindings, n]
+        )
+      ).rows as any[]
+    const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
+    if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
+    if (rows.length < limit) {
+      rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
+    }
     return rows
   }
 
-  /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
+  /** Spots posted before this are listed, not imported: their NZB is read on first download. */
+  importCutoff(): Date {
+    return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
+  }
+
+  /** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
+  isListable(spot: any): boolean {
+    return (
+      spot.verified === true &&
+      spot.posted_at !== null &&
+      new Date(spot.posted_at) < this.importCutoff() &&
+      !moderationTarget({ keyId: spot.key_id, title: spot.title })
+    )
+  }
+
+  /**
+   * Creates the listing releases (source `spot`, no files) for these spots.
+   * The release key is the one an import of the spot's NZB uses, so the
+   * import on first download fills this release.
+   */
+  async list(spots: any[]): Promise<number> {
+    if (spots.length === 0) return 0
+    const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
+    const ids = await IngestService.upsertReleases(
+      db.connection().getWriteClient(),
+      spots.map((spot) => ({
+        release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
+        poster: spot.poster,
+        groups: [spot.group_name],
+        postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
+      }))
+    )
+    const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
+    await db.rawQuery(
+      `UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
+       WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
+      [JSON.stringify(rows)]
+    )
+    await db.rawQuery(
+      `UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
+       FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
+       WHERE s.message_id = v.message_id`,
+      [JSON.stringify(rows)]
+    )
+    return rows.length
+  }
+
+  /**
+   * Verifies the spot on its raw header (the signature covers ISO-8859-1
+   * bytes, which the overview, read as UTF-8, may have mangled), then
+   * applies moderation, lists an old spot or imports a recent one's NZB.
+   * Returns the new status.
+   */
   async process(spot: any): Promise<string> {
     const update: Record<string, unknown> = { processed_at: new Date(), error: null }
     try {
-      const raw = await this.fetch(`HEAD <${spot.message_id}>`)
-      const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
-      // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
-      const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
-      if (!header || !verifySpotHeader(header)) {
+      const head = await this.readHead(spot.message_id)
+      if (!head) {
         update.status = 'unverified'
-      } else if (moderationTarget(header)) {
+      } else if (moderationTarget(head.header)) {
         update.status = 'moderation'
-        await this.moderate(header, moderationTarget(header)!)
+        await this.moderate(head.header, moderationTarget(head.header)!)
       } else {
-        const parsed = parseSpotXml(xml)
-        update.title = header.title
-        update.poster = header.poster
-        update.spotter_id = spotterId(header.selfSignedPubKey) || null
-        if (parsed?.website) update.website = parsed.website
-        const segments = parsed?.nzbSegments ?? []
-        if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
-          update.status = 'no-nzb'
+        update.title = spotTitle(head)
+        update.poster = head.header.poster
+        update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
+        Object.assign(update, this.details(head.xml))
+        if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
+          await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
+          update.status = 'listed'
         } else {
-          const bodies = []
-          for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
-          const nzbXml = decodeSpotBinary(bodies).toString('utf8')
-          const nzb = parseNzb(nzbXml)
-          if (!nzb || nzb.files.length === 0) {
-            update.status = 'no-nzb'
-          } else {
-            const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
-            const result = await NzbImportService.import(
-              {
-                id: null,
-                filename: `${header.title}.nzb`,
-                poster: header.poster,
-                groups,
-                date: header.stamp * 1000,
-                message_ids: { '1': { id: spot.message_id } },
-              },
-              nzbXml,
-              { name: header.title, source: 'spot', moveContainer: false }
-            )
-            update.status = result ? 'imported' : 'no-nzb'
-            if (result) update.release_id = result.releaseId
-          }
+          Object.assign(update, await this.importNzb(spot.message_id, head))
         }
       }
     } catch (error: any) {
@@ -131,9 +199,149 @@ class SpotService {
       logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
     }
     await db.from('spots').where('message_id', spot.message_id).update(update)
+    if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
     return update.status as string
   }
 
+  /**
+   * The NZB of a listing, on first download: reads and imports it. Returns
+   * the release that now holds its files (the listing's own, unless the NZB
+   * joined files already known elsewhere), or null.
+   */
+  async importListing(releaseId: number): Promise<number | null> {
+    const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
+    if (!spot) return null
+    return this.reimport(spot)
+  }
+
+  /**
+   * Reads a spot's NZB again and imports it, so its files end up in the
+   * spot's own release under the spot's title. Returns that release, or null.
+   */
+  async reimport(spot: any): Promise<number | null> {
+    const head = await this.readHead(spot.message_id)
+    if (!head) return null
+    const result = await this.importNzb(spot.message_id, head)
+    await db
+      .from('spots')
+      .where('message_id', spot.message_id)
+      .update({ ...result, ...this.details(head.xml), processed_at: new Date() })
+    await this.applyPassword(spot.message_id)
+    return (result.release_id as number | undefined) ?? null
+  }
+
+  /**
+   * A password the spotter wrote in the spot's title or description (not in
+   * the NZB) goes on its release, unless the release has one already.
+   */
+  async applyPassword(messageId: string): Promise<string | null> {
+    const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
+    const password = spot ? passwordFromSpot(spot.title, spot.description) : null
+    if (!password) return null
+    await db
+      .from('releases')
+      .where('id', spot.release_id)
+      .whereNull('password')
+      .update({ password, password_source: 'spot' })
+    return password
+  }
+
+  /** Columns for what the spot's XML adds to its overview: description, image, website. */
+  private details(xml: SpotXml | null): Record<string, unknown> {
+    return {
+      description: xml?.description || null,
+      image: xml?.image ? JSON.stringify(xml.image) : null,
+      ...(xml?.website ? { website: xml.website } : {}),
+      details_at: new Date(),
+    }
+  }
+
+  /**
+   * A spot row with its description and image, reading its full header
+   * first when that hasn't happened yet (old spots are listed from their
+   * overview only). Returns the row as it is when Usenet can't be reached.
+   */
+  async withDetails(spot: any): Promise<any> {
+    if (spot.details_at) return spot
+    try {
+      const head = await this.readHead(spot.message_id)
+      const update = head ? this.details(head.xml) : { details_at: new Date() }
+      await db.from('spots').where('message_id', spot.message_id).update(update)
+      await this.applyPassword(spot.message_id)
+      return { ...spot, ...update, image: head?.xml?.image ?? null }
+    } catch (error: any) {
+      logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
+      // The article is gone (430): nothing to read later either.
+      if (/^4\d\d/.test(String(error.message))) {
+        await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
+      }
+      return spot
+    }
+  }
+
+  /**
+   * Reads the details of up to `limit` imported spots that don't have them
+   * (imported before details were kept), newest first. Returns how many.
+   * Listed spots are left to their first view: there are millions.
+   * Returns how many were read (0 when Usenet can't be reached).
+   */
+  async fillDetails(limit: number): Promise<number> {
+    const spots = await db
+      .from('spots')
+      .where('status', 'imported')
+      .whereNull('details_at')
+      .orderBy('posted_at', 'desc')
+      .limit(limit)
+    let read = 0
+    for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
+    return read
+  }
+
+  /** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
+  async image(spot: any): Promise<Buffer | null> {
+    const segments: string[] = spot.image?.segments ?? []
+    if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
+    const bodies = []
+    for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+    return decodeSpotBinary(bodies, false)
+  }
+
+  /** The spot's raw header, verified; null when it doesn't verify. */
+  private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
+    const raw = await this.fetch(`HEAD <${messageId}>`)
+    const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
+    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
+    if (!header || !verifySpotHeader(header)) return null
+    return { header, xml: parseSpotXml(decodeSpotText(xml)) }
+  }
+
+  /** Downloads the spot's compressed NZB and imports it as a release (D26). */
+  private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
+    const segments = head.xml?.nzbSegments ?? []
+    if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
+    const bodies = []
+    for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+    const nzbXml = decodeSpotBinary(bodies).toString('utf8')
+    const nzb = parseNzb(nzbXml)
+    if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
+    const { header } = head
+    if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
+    const title = spotTitle(head)
+    const result = await NzbImportService.import(
+      {
+        id: null,
+        filename: `${title}.nzb`,
+        poster: header.poster,
+        groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
+        date: header.stamp * 1000,
+        first_message_id: messageId,
+      },
+      nzbXml,
+      { name: title, source: 'spot', moveContainer: false, trusted: true }
+    )
+    return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
+  }
+
   /**
    * Applies a delete/dispose/remove: the target spot is marked `removed`
    * (its release is then hidden from search). As in Spotweb, a personal
@@ -150,6 +358,15 @@ class SpotService {
     }
     await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
     logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+    // Spots that joined its release (a correction, a repost) take the files over now.
+    if (target.release_id) {
+      const sharing = await db
+        .from('spots')
+        .where('release_id', target.release_id)
+        .where('status', 'imported')
+        .whereNot('message_id', targetId)
+      for (const spot of sharing) await this.reimport(spot)
+    }
   }
 
   /** One HEAD or BODY, as raw bytes. */
@@ -157,7 +374,7 @@ class SpotService {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      const { data } = await conn._artcmd(command)
+      const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
       return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
     } catch (error: any) {
       // 430 (no such article) is an answer, not a broken connection.

app/services/StagingService.ts

@@ -0,0 +1,252 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import indexerConfig from '#config/indexer'
+import IngestService, { type IngestSegment } from '#services/IngestService'
+import { counterFileKey } from '#lib/subject/index'
+import { fileStem } from '#lib/release/index'
+
+/** Rows per statement (a key and a JSON array each). */
+const CHUNK = 5000
+
+/**
+ * Staging of lone header segments (CONCEPT.md D35). Uploads that give every
+ * article its own random subject turned each article into a binary, a part
+ * and a release, which GC deleted two hours later: ~3.9M binaries an hour
+ * against ~123k real files. A segment whose file is new and alone in its
+ * fetch batch now waits in `staged_segments` (unlogged, one index) until a
+ * second segment of that file arrives; only then do both go into the real
+ * tables. GC drops what waited longer than STAGING_TTL_SECONDS (30 min).
+ *
+ * Counter grouping (D36): those random-subject uploads still keep poster,
+ * file counter `[i/M]` and part count per file. Staged segments also carry
+ * that key (`counter_key`); once two staged segments from different subjects
+ * share it, they become one file under that key, and later segments with it
+ * join that file directly.
+ */
+class StagingService {
+  enabled(groupName: string): boolean {
+    const groups = indexerConfig.stagingGroups
+    return groups.includes('*') || groups.includes(groupName)
+  }
+
+  /** Whether counter grouping (D36) applies to this group. */
+  countersEnabled(groupName: string): boolean {
+    const groups = indexerConfig.counterGroupingGroups
+    return groups.includes('*') || groups.includes(groupName)
+  }
+
+  /**
+   * The segments of a header batch that should be ingested now: the batch's
+   * own, minus lone segments of new files (staged instead), plus staged
+   * segments whose file this batch shows to be real. `staged` counts the
+   * segments that went to staging.
+   */
+  async filter(segments: IngestSegment[], { counters = false } = {}): Promise<{ ingest: IngestSegment[]; staged: number }> {
+    const byKey = new Map<string, IngestSegment[]>()
+    for (const segment of segments) {
+      const list = byKey.get(segment.fileKey)
+      if (list) list.push(segment)
+      else byKey.set(segment.fileKey, [segment])
+    }
+
+    const ingest: IngestSegment[] = []
+    const lone: IngestSegment[] = []
+    for (const list of byKey.values()) {
+      // Single-segment files are complete; two different articles of a file in one batch make it real.
+      // (One article twice is a crosspost, LOG.md 2026-10-02.)
+      const articles = new Set(list.map((s) => s.messageId)).size
+      if (list[0].totalParts <= 1 || articles >= 2 || list[0].weakIdentity || list[0].size != null) ingest.push(...list)
+      else lone.push(list[0])
+    }
+
+    const counterOf = new Map<IngestSegment, string>()
+    if (counters) {
+      for (const segment of lone) {
+        const key = counterFileKey(segment)
+        if (key) counterOf.set(segment, key)
+      }
+    }
+
+    // Files we already have a binary for (by subject, or by counters) take their segments directly.
+    const known = new Set<string>()
+    const lookup = [...new Set([...lone.map((s) => s.fileKey), ...counterOf.values()])]
+    for (let i = 0; i < lookup.length; i += CHUNK) {
+      const rows = await db.from('binaries').whereIn('file_key', lookup.slice(i, i + CHUNK)).select('file_key')
+      for (const row of rows) known.add(row.file_key)
+    }
+    const toStage: IngestSegment[] = []
+    const knownCounters = new Set<string>()
+    for (const segment of lone) {
+      const counter = counterOf.get(segment)
+      if (known.has(segment.fileKey)) {
+        ingest.push(segment)
+      } else if (counter && known.has(counter)) {
+        ingest.push(asCounterFile(segment, counter))
+        knownCounters.add(counter)
+      } else {
+        toStage.push(segment)
+      }
+    }
+
+    // Stage the rest; a file that now has two staged segments comes out again.
+    const promoted: string[] = []
+    // Key order, so concurrent jobs lock rows in the same order.
+    toStage.sort((a, b) => a.fileKey.localeCompare(b.fileKey))
+    for (let i = 0; i < toStage.length; i += CHUNK) {
+      const rows = toStage.slice(i, i + CHUNK).map((s) => ({
+        file_key: s.fileKey,
+        counter_key: counterOf.get(s) ?? null,
+        segments: [serialize(s)],
+        staged_at: new Date(),
+      }))
+      const { rows: result } = await retryDeadlock(() => db.rawQuery(
+        `INSERT INTO staged_segments (file_key, counter_key, segments, staged_at)
+         SELECT file_key, counter_key, segments, staged_at
+         FROM jsonb_to_recordset(?::jsonb) AS v(file_key text, counter_key text, segments jsonb, staged_at timestamptz)
+         ON CONFLICT (file_key) DO UPDATE SET segments = CASE
+           -- The same article again (a crosspost, or a re-fetched range) is no second segment.
+           WHEN EXISTS (SELECT 1 FROM jsonb_array_elements(staged_segments.segments) e
+                        WHERE e->>'messageId' = EXCLUDED.segments->0->>'messageId') THEN staged_segments.segments
+           ELSE staged_segments.segments || EXCLUDED.segments END
+         RETURNING file_key, jsonb_array_length(segments) AS n`,
+        [JSON.stringify(rows)]
+      ))
+      for (const row of result) if (Number(row.n) >= 2) promoted.push(row.file_key)
+    }
+
+    // Counter files: two staged rows (two subjects) with one counter key make a file.
+    const stagedCounters = [...new Set(toStage.map((s) => counterOf.get(s)).filter((k): k is string => Boolean(k)))]
+    const readyCounters: string[] = []
+    for (let i = 0; i < stagedCounters.length; i += CHUNK) {
+      const { rows } = await db.rawQuery(
+        `SELECT counter_key FROM staged_segments WHERE counter_key = ANY(?) GROUP BY counter_key HAVING count(*) >= 2`,
+        [stagedCounters.slice(i, i + CHUNK)]
+      )
+      for (const row of rows) readyCounters.push(row.counter_key)
+    }
+
+    // Promoted files, staged segments of files this batch ingests anyway, and ready counter files.
+    const fileKeys = [...new Set([...promoted, ...ingest.map((s) => s.fileKey)])]
+    const counterKeys = [...new Set([...readyCounters, ...knownCounters])]
+    const seen = new Set(ingest.map((s) => s.messageId))
+    for (const segment of await this.take(fileKeys, counterKeys)) {
+      if (!seen.has(segment.messageId)) ingest.push(segment)
+      seen.add(segment.messageId)
+    }
+    const staged = toStage.filter((s) => !seen.has(s.messageId)).length
+    return { ingest, staged }
+  }
+
+  /**
+   * GC: drops staged segments that waited longer than `maxAgeSeconds`, then
+   * ingests those whose file got a binary meanwhile (two fetch jobs racing).
+   * Expiry comes first so the table stays bounded whatever the budget; the
+   * adoption starts from recently seen binaries (index on last_seen_at,
+   * which moves in 10-minute steps) rather than joining every staged row.
+   */
+  async sweep(maxAgeSeconds: number, budget: number): Promise<{ adopted: number; expired: number }> {
+    let adopted = 0
+    let expired = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM staged_segments WHERE ctid IN (
+           SELECT ctid FROM staged_segments WHERE staged_at < now() - make_interval(secs => ?) LIMIT 20000)`,
+        [maxAgeSeconds]
+      )
+      expired += result.rowCount
+      if (result.rowCount < 20000) break
+    }
+    let lastId = 0
+    while (Date.now() < budget) {
+      const recent = await db
+        .from('binaries')
+        .where('last_seen_at', '>', new Date(Date.now() - (maxAgeSeconds + 15 * 60) * 1000))
+        .where('id', '>', lastId)
+        .orderBy('id')
+        .limit(CHUNK)
+        .select('id', 'file_key')
+      if (recent.length === 0) break
+      lastId = Number(recent[recent.length - 1].id)
+      const keys = recent.map((b) => b.file_key as string)
+      // A binary's key is a subject key or a counter key; staged rows are looked up by both.
+      const segments = await this.take(keys, keys)
+      if (segments.length > 0) await IngestService.ingest(segments)
+      adopted += segments.length
+    }
+    return { adopted, expired }
+  }
+
+  /**
+   * Removes and returns the staged segments of these files: by subject key,
+   * or by counter key (returned as segments of the counter file).
+   */
+  private async take(fileKeys: string[], counterKeys: string[] = []): Promise<IngestSegment[]> {
+    const taken: IngestSegment[] = []
+    for (let i = 0; i < fileKeys.length; i += CHUNK) {
+      const { rows } = await retryDeadlock(() =>
+        db.rawQuery(`DELETE FROM staged_segments WHERE file_key = ANY(?) RETURNING segments`, [fileKeys.slice(i, i + CHUNK)])
+      )
+      for (const row of rows) taken.push(...(row.segments as any[]).map(deserialize))
+    }
+    for (let i = 0; i < counterKeys.length; i += CHUNK) {
+      const { rows } = await retryDeadlock(() =>
+        db.rawQuery(`DELETE FROM staged_segments WHERE counter_key = ANY(?) RETURNING counter_key, segments`, [counterKeys.slice(i, i + CHUNK)])
+      )
+      for (const row of rows) taken.push(...(row.segments as any[]).map((s) => asCounterFile(deserialize(s), row.counter_key)))
+    }
+    return taken
+  }
+}
+
+/**
+ * A segment as part of its counter file (D36): the counter key is its file
+ * key, it is a weak identity (a repeated part number from another upload
+ * marks the binary conflicted), and its release is a weak one of its own,
+ * which the upload's par2 set can gather later.
+ */
+function asCounterFile(segment: IngestSegment, counterKey: string): IngestSegment {
+  return {
+    ...segment,
+    fileKey: counterKey,
+    weakIdentity: true,
+    release: {
+      key: createHash('sha1').update(`counter-release\n${counterKey}`).digest('hex'),
+      name: fileStem(segment.name),
+      source: 'weak-stem',
+    },
+  }
+}
+
+/**
+ * Runs a staging statement again when it lost a deadlock: concurrent fetch
+ * jobs insert by file key and delete by counter key, so their row locks can
+ * cross. Postgres rolled the statement back, and each one is safe to repeat.
+ */
+async function retryDeadlock<T>(statement: () => Promise<T>, attempts = 5): Promise<T> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await statement()
+    } catch (error: any) {
+      if (error?.code !== '40P01' || attempt >= attempts) throw error
+      await new Promise((resolve) => setTimeout(resolve, 50 * attempt + Math.random() * 100))
+    }
+  }
+}
+
+function serialize(segment: IngestSegment) {
+  return {
+    ...segment,
+    postedAt: segment.postedAt?.toISOString(),
+    articleNumber: segment.articleNumber?.toString(),
+  }
+}
+
+function deserialize(stored: any): IngestSegment {
+  return {
+    ...stored,
+    postedAt: stored.postedAt ? new Date(stored.postedAt) : undefined,
+    articleNumber: stored.articleNumber ?? undefined,
+  }
+}
+
+export default new StagingService()

commands/ContentBackfill.ts

@@ -3,6 +3,7 @@ import { CommandOptions } from '@adonisjs/core/types/ace'
 import db from '@adonisjs/lucid/services/db'
 import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
+import { isArchiveStart } from '#lib/content/index'
 
 const BATCH = 1000
 
@@ -28,17 +29,37 @@ export default class ContentBackfill extends BaseCommand {
   @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
   declare archiveNzbs: boolean
 
+  @flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
+  declare sevenZip: boolean
+
+  @flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
+  declare archives: boolean
+
+  @flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
+  declare sfv: boolean
+
+  @flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
+  declare nicovideo: boolean
+
   public async run() {
     let lastId = 0
     let queued = 0
     try {
       while (!this.limit || queued < this.limit) {
         // Files never inspected: no fingerprint on their binary and no archive listing.
-        const query = db
-          .from('files as f')
-          .join('binaries as b', 'b.file_id', 'f.id')
-          .where('f.id', '>', lastId)
-        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
+        const query = db.from('files as f').where('f.id', '>', lastId)
+        // Files from NZBs (spots) have no binary.
+        if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
+        else query.join('binaries as b', 'b.file_id', 'f.id')
+        if (this.nicovideo) {
+          query.whereNull('f.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
+        } else if (this.sfv) {
+          query.whereNull('f.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
+        } else if (this.archives) {
+          query.whereNull('f.content_type').whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
+        } else if (this.sevenZip) {
+          query.whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
+        } else if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
         else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
         const files = await query
           .orderBy('f.id')
@@ -48,14 +69,19 @@ export default class ContentBackfill extends BaseCommand {
         lastId = Number(files.at(-1).id)
 
         const jobs = files
-          .filter((f) => ContentService.isCandidate(f.filename))
+          .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
           .slice(0, this.limit ? this.limit - queued : undefined)
           .map((f) => ({
             name: 'process-collection',
             data: { fileId: Number(f.id) },
             // A new id per run: BullMQ ignores an add() for an id it still knows.
             // Targeted backfills go ahead of whatever is waiting.
-            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
+            opts: {
+              jobId: `content-${f.id}-${Date.now()}`,
+              lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
+              // niconico captures are many (~190k) and only cosmetic: behind everything else.
+              ...(this.nicovideo ? { priority: 10 } : {}),
+            },
           }))
         if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
         queued += jobs.length

commands/DbReindex.ts

@@ -0,0 +1,22 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import IndexMaintenance from '#services/IndexMaintenance'
+
+/** Rebuilds bloated indexes online (gc:scheduler also does this weekly). */
+export default class DbReindex extends BaseCommand {
+  public static commandName = 'db:reindex'
+  public static description = 'Rebuilds B-tree indexes that are mostly empty space (online, REINDEX CONCURRENTLY).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show which indexes would be rebuilt' })
+  declare dryRun: boolean
+
+  public async run() {
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`${rebuilt.length} indexes ${this.dryRun ? 'would be rebuilt' : `rebuilt, ${Math.round(saved / 1024 / 1024)} MB freed`}.`)
+  }
+}

commands/FilesCleanNames.ts

@@ -0,0 +1,35 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { unquotedFileName } from '#services/NzbImportService'
+
+/**
+ * Files named after a whole NZB subject without quotes (spot NZBs), e.g.
+ * `Adventure.part0055.rar - [056/175] (1/1)`: renamed to the filename in it.
+ */
+export default class FilesCleanNames extends BaseCommand {
+  public static commandName = 'files:clean-names'
+  public static description = 'Strips subject leftovers (counters, "yEnc") from file names that came from NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show what would change' })
+  declare dryRun: boolean
+
+  public async run() {
+    const files = await db
+      .from('files')
+      .whereRaw(`filename ~* '(\\(\\d+/\\d+\\)|yEnc|\\[\\d+/\\d+\\])\\s*$'`)
+      .select('id', 'filename')
+    const changes = files
+      .map((f) => ({ id: Number(f.id), from: f.filename as string, to: unquotedFileName(f.filename) }))
+      .filter((c) => c.to !== c.from)
+    for (const c of changes.slice(0, 5)) this.logger.info(`"${c.from}" → "${c.to}"`)
+    if (!this.dryRun) {
+      for (const c of changes) await db.from('files').where('id', c.id).update({ filename: c.to })
+    }
+    this.logger.info(`${changes.length} of ${files.length} file names ${this.dryRun ? 'would change' : 'cleaned'}.`)
+  }
+}

commands/GcScheduler.ts

@@ -5,6 +5,9 @@ import db from '@adonisjs/lucid/services/db'
 import gcConfig from '#config/gc'
 import QueueService from '#services/QueueService'
 import RedisService from '#services/RedisService'
+import IndexMaintenance from '#services/IndexMaintenance'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+import StagingService from '#services/StagingService'
 import Shutdown from '#services/Shutdown'
 
 const CLEAN_BATCH = 1000
@@ -63,13 +66,20 @@ export default class GcScheduler extends BaseCommand {
     this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
 
     if (!this.dryRun) {
-      const budget = Date.now() + gcConfig.deleteBudgetMs
-      const parts = await this.dropCopiedParts(budget)
-      const single = await this.deleteSingleArticleBinaries(budget)
-      const releases = await this.deleteEmptyReleases(budget)
-      if (parts || single || releases) {
+      // Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
+      const budget = () => Date.now() + gcConfig.deleteBudgetMs
+      const parts = await this.dropCopiedParts(budget())
+      const single = await this.deleteSingleArticleBinaries(budget())
+      const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
+      const releases = await this.deleteEmptyReleases(budget())
+      const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
+      if (staging.adopted || staging.expired) {
+        this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
+      }
+      if (parts || single || unfingerprinted || releases) {
         this.logger.info(
-          `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.`
+          `Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
+            `${releases} empty releases deleted.`
         )
       }
     }
@@ -83,6 +93,23 @@ export default class GcScheduler extends BaseCommand {
         `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
       )
     }
+
+    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep.
+    const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+
+    await this.rebuildIndexes()
+  }
+
+  /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
+  private async rebuildIndexes() {
+    const key = 'gc:reindex:last'
+    const last = Number((await RedisService.client.get(key)) ?? 0)
+    if (Date.now() - last < gcConfig.reindexIntervalMs) return
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
   }
 
   /**
@@ -151,10 +178,11 @@ export default class GcScheduler extends BaseCommand {
     let total = 0
     while (Date.now() < budget) {
       const result = await db.rawQuery(
-        `DELETE FROM parts WHERE binary_id IN (
+        // Header-derived parts (no per-part CRC) of binaries that are a file now.
+        `DELETE FROM parts WHERE pcrc32 IS NULL AND binary_id IN (
            SELECT b.id FROM binaries b
-           WHERE b.file_id IS NOT NULL AND b.size IS NULL
-             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id)
+           WHERE b.file_id IS NOT NULL
+             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id AND p.pcrc32 IS NULL)
            LIMIT ?)`,
         [500]
       )
@@ -190,6 +218,29 @@ export default class GcScheduler extends BaseCommand {
     return total
   }
 
+  /**
+   * Completed header-derived binaries without a fingerprint, some hours after
+   * completion: their file exists, their parts are gone, and par2 naming needs
+   * a size and MD5 they don't have. Oldest first, via the last_seen_at index.
+   */
+  private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM binaries WHERE id IN (
+           SELECT id FROM binaries
+           WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
+             AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
+           ORDER BY last_seen_at LIMIT ?)`,
+        [cutoff, gcConfig.deleteBatch]
+      )
+      total += result.rowCount
+      if (result.rowCount < gcConfig.deleteBatch) break
+    }
+    return total
+  }
+
   /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
   private async deleteEmptyReleases(budget: number): Promise<number> {
     const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)

commands/GroupAdd.ts

@@ -33,6 +33,9 @@ export default class GroupAdd extends BaseCommand {
   @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
   declare reset: boolean
 
+  @flags.boolean({ description: "Start at the server's oldest article: the group's full history (also for groups indexed before)" })
+  declare fromStart: boolean
+
   public async run() {
     for (const name of this.names) {
       if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
@@ -42,26 +45,30 @@ export default class GroupAdd extends BaseCommand {
       const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
       group.active = !this.disable
       if (this.bodies) group.fetchBodies = true
-      if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
-        // The scheduler continues after last_queued_id.
-        const last = await this.lastArticle(name)
+      if (this.fromStart) {
+        // The scheduler continues after last_queued_id; 0 would mean "new group" to it.
+        const { first } = await this.groupInfo(name)
+        group.lastQueuedId = first > 1n ? first - 1n : 1n
+        group.lastIndexedId = group.lastQueuedId
+      } else if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
+        const { last } = await this.groupInfo(name)
         group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
         group.lastIndexedId = group.lastQueuedId
       }
       await group.save()
       this.logger.info(
         `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
-          (this.backfill !== undefined ? `, starting after article ${group.lastQueuedId}` : '')
+          (this.backfill !== undefined || this.fromStart ? `, starting after article ${group.lastQueuedId}` : '')
       )
     }
-    if (this.backfill !== undefined) await NntpService.shutdown()
+    if (this.backfill !== undefined || this.fromStart) await NntpService.shutdown()
   }
 
-  private async lastArticle(name: string): Promise<bigint> {
+  private async groupInfo(name: string) {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      return parseGroupResponse((await conn.group(name)).resp).last
+      return parseGroupResponse((await conn.group(name)).resp)
     } catch (error) {
       failed = true
       throw error

commands/GroupBackfill.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import BackfillService from '#services/BackfillService'
+import { isSpotGroup } from '#services/SpotService'
+
+/**
+ * Indexes a group's older headers (CONCEPT.md D32): from where live
+ * indexing began back to N days ago, header-only and behind live indexing.
+ * index:scheduler queues the ranges; this only sets where to go.
+ */
+export default class GroupBackfill extends BaseCommand {
+  public static commandName = 'group:backfill'
+  public static description = "Indexes groups' older headers (header-only, behind live indexing)."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Groups (default: every active binary group)', required: false })
+  declare names?: string[]
+
+  @flags.number({ description: 'How many days back', default: 21 })
+  declare days: number
+
+  @flags.boolean({ description: 'Stop backfilling these groups (ranges already queued still run)' })
+  declare stop: boolean
+
+  public async run() {
+    const names = this.names?.length
+      ? this.names
+      : (await db.from('groups').where('active', true).orderBy('name').select('name')).map((g) => g.name as string).filter((n) => !isSpotGroup(n))
+    try {
+      for (const name of names) {
+        if (this.stop) {
+          await BackfillService.stop(name)
+          this.logger.info(`${name}: backfill stopped.`)
+          continue
+        }
+        const range = await BackfillService.start(name, this.days)
+        this.logger.info(
+          range
+            ? `${name}: backfilling articles ${range.from}–${range.to} (${(range.to - range.from).toLocaleString('en')} articles, ${this.days} days).`
+            : `${name}: nothing to backfill.`
+        )
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/IndexScheduler.ts

@@ -7,6 +7,7 @@ import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
 import FetchRangeService from '#services/FetchRangeService'
 import SplitService from '#services/SplitService'
+import BackfillService from '#services/BackfillService'
 
 export default class IndexScheduler extends BaseCommand {
   public static commandName = 'index:scheduler'
@@ -107,9 +108,19 @@ export default class IndexScheduler extends BaseCommand {
       }
     }
 
+    const backfill = async () => {
+      try {
+        const queued = await BackfillService.tick()
+        if (queued > 0) this.logger.info(`Queued ${queued} backfill range(s).`)
+      } catch (err: any) {
+        this.logger.error(`Error queuing backfill: ${err.message}`)
+      }
+    }
+
     // Run immediately, then wait a full interval after each run so runs never overlap.
     while (!Shutdown.requested) {
       await schedule()
+      await backfill()
       await split()
       await Shutdown.sleep(indexerConfig.intervalMs)
     }

commands/ReleaseMergeSplit.ts

@@ -0,0 +1,21 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+
+/** Merges uploads split per poster (gc:scheduler also does this every sweep). */
+export default class ReleaseMergeSplit extends BaseCommand {
+  public static commandName = 'release:merge-split'
+  public static description = 'Merges releases of one upload that was posted under a different poster per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const { names, releases } = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    this.logger.info(`${names} split uploads; ${releases} releases ${this.dryRun ? 'would be' : ''} merged into their first.`)
+  }
+}

commands/SpotPasswords.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import SpotService from '#services/SpotService'
+
+/** Passwords from the titles and descriptions of spots read so far, onto their releases. */
+export default class SpotPasswords extends BaseCommand {
+  public static commandName = 'spot:passwords'
+  public static description = "Takes archive passwords from spots' titles and descriptions onto their releases."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const spots = await db
+      .from('spots')
+      .whereNotNull('release_id')
+      .where((q) => q.whereNotNull('description').orWhere('title', 'like', '%{{%}}%'))
+      .select('message_id')
+    let found = 0
+    for (const spot of spots) if (await SpotService.applyPassword(spot.message_id)) found++
+    this.logger.info(`${found} of ${spots.length} spots name a password in their title or description.`)
+  }
+}

commands/SpotReimport.ts

@@ -0,0 +1,51 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+/**
+ * Imported spots whose files sit in a release that isn't the spot's own
+ * (another source, or another name): their NZB is read again, which moves
+ * the files into the spot's release under its title (CONCEPT.md D29).
+ */
+export default class SpotReimport extends BaseCommand {
+  public static commandName = 'spot:reimport'
+  public static description = "Moves imported spots' files into their own release, named after the spot."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const spots = await db
+      .from('spots as s')
+      .join('releases as r', 'r.id', 's.release_id')
+      .where('s.status', 'imported')
+      .where((q) => q.whereNot('r.source', 'spot').orWhereRaw('r.name <> s.title'))
+      .orderBy('s.posted_at', 'desc')
+      .select('s.*', 'r.name as release_name')
+    this.logger.info(`${spots.length} imported spots are in a release that isn't their own.`)
+    if (this.dryRun) return
+    let moved = 0
+    let failed = 0
+    try {
+      for (const spot of spots) {
+        try {
+          const releaseId = await SpotService.reimport(spot)
+          if (releaseId) moved++
+          else failed++
+        } catch (error: any) {
+          failed++
+          this.logger.warning(`${spot.title}: ${error.message}`)
+        }
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+    this.logger.info(`${moved} spots re-imported, ${failed} without a usable NZB (dummy, gone) or failed.`)
+  }
+}

commands/workers/Collection.ts

@@ -49,6 +49,10 @@ export default class CollectionWorker extends BaseCommand {
         if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
         if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
         if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
+        if (content?.yencName) this.logger.info(`"${file.filename}" is "${content.yencName}" (yEnc name).`)
+        if (content?.nicovideo) this.logger.info(`"${file.filename}" → "${content.nicovideo}".`)
+        if (content?.sfvMoved) this.logger.info(`SFV "${file.filename}" moved ${content.sfvMoved} files into its release.`)
+        if (content?.contentType === 'yencbin') this.logger.info(`"${file.filename}" is a yEncBin encrypted container.`)
         if (content?.nzb) {
           const n = content.nzb
           this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)

commands/workers/Fetch.ts

@@ -12,6 +12,7 @@ import FetchRangeService from '#services/FetchRangeService'
 import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
 import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
 import SpotService, { isSpotGroup } from '#services/SpotService'
+import StagingService from '#services/StagingService'
 
 export default class FetchWorker extends BaseCommand {
   public static commandName = 'worker:fetch'
@@ -36,8 +37,10 @@ export default class FetchWorker extends BaseCommand {
       let failed = false
       try {
         conn = await pool.acquire()
-        await conn.group(groupName)
-        overviews = (await conn.xover(startId, endId)).overviews
+        await pool.timed(conn.group(groupName), `GROUP ${groupName}`)
+        const reply = await pool.timed<any>(conn.xover(startId, endId), `XOVER ${groupName} ${startId}-${endId}`)
+        // An empty reply parses as one line without fields ([null, {}]).
+        overviews = reply.overviews.filter(([number, header]: [number | null, Record<string, string>]) => number !== null && header?.['message-id'])
       } catch (error: any) {
         failed = true
         this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
@@ -56,7 +59,8 @@ export default class FetchWorker extends BaseCommand {
       }
 
       const group = await db.from('groups').where('name', groupName).first()
-      const fetchBodies = Boolean(group?.fetch_bodies)
+      // Backfill is header-only: old bodies would be far more than the link carries (CONCEPT.md D32).
+      const fetchBodies = Boolean(group?.fetch_bodies) && !job.data.backfill
 
       const segments: IngestSegment[] = []
       const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
@@ -72,7 +76,11 @@ export default class FetchWorker extends BaseCommand {
         else obfuscated.push({ articleNumber, header })
       }
 
-      const completed = await IngestService.ingest(segments)
+      // Lone segments of new files wait in staging until their file proves real (D35).
+      const { ingest, staged } = StagingService.enabled(groupName)
+        ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(groupName) })
+        : { ingest: segments, staged: 0 }
+      const completed = await IngestService.ingest(ingest)
       const deferredFiles = await DeferredService.record(groupName, deferred)
 
       if (obfuscated.length > 0) {
@@ -82,7 +90,7 @@ export default class FetchWorker extends BaseCommand {
       if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
 
       this.logger.info(
-        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${segments.length} stored, ` +
+        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${ingest.length} stored${staged ? `, ${staged} staged` : ''}, ` +
           `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
           `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
       )

commands/workers/File.ts

@@ -2,6 +2,9 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
 import queueConfig from '#config/queue'
 import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
@@ -42,15 +45,10 @@ export default class FileWorker extends BaseCommand {
         throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
       }
 
-      const messageIds = Object.fromEntries(
-        parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
-      )
+      const segments = parts.map((part) => ({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) }))
 
       // Already known as a file, e.g. added from a posted NZB before its own articles completed.
-      const existing = await db
-        .from('files')
-        .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
-        .first()
+      const existing = await db.from('files').where('first_message_id', segments[0]?.id ?? '').first()
       if (existing) {
         await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
         await dropCopiedParts(binary)
@@ -59,26 +57,23 @@ export default class FileWorker extends BaseCommand {
       }
 
       // file_key is unique: a second completion of the same upload inserts nothing.
-      const knex = db.connection().getWriteClient()
       const now = new Date()
-      const [inserted] = await knex('files')
-        .insert({
+      const [inserted] = await FileStore.insert([
+        {
           file_key: binary.file_key,
           filename: binary.name,
           poster: binary.poster,
           date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
           parts: parts.length,
-          message_ids: JSON.stringify(messageIds),
+          segments,
           groups: JSON.stringify(binary.groups),
           release_id: binary.release_id,
           real_name: binary.real_name,
           size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
           created_at: now,
           updated_at: now,
-        })
-        .onConflict('file_key')
-        .ignore()
-        .returning('id')
+        },
+      ])
 
       const fileId =
         inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
@@ -91,8 +86,21 @@ export default class FileWorker extends BaseCommand {
       }
 
       this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
+      // An SFV posted earlier may already list it (D34).
+      await SfvService.claim({
+        id: Number(fileId),
+        name: binary.real_name ?? binary.name,
+        poster: binary.poster,
+        date: binary.posted_at ? new Date(binary.posted_at).getTime() : null,
+        release_id: binary.release_id,
+      })
       await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and
+      // the file insert ignores duplicates, so they can run side by side.
+      concurrency: env.get('FILE_CONCURRENCY', 8),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`File job ${job?.id} failed: ${err.message}`)
@@ -112,6 +120,13 @@ export default class FileWorker extends BaseCommand {
  * ones keep them, so articles of an equal-sized next volume are recognised
  * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
  */
-async function dropCopiedParts(binary: { id: number; size: number | null }) {
-  if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
+/**
+ * The file holds the segment list now. Header-derived parts (no per-part
+ * CRC) aren't needed any more; body-derived ones are kept for split-volume
+ * verification (D20). Going by the CRC, not the binary's size: content
+ * inspection gives header-derived binaries a size too, which kept ~20M
+ * parts around (LOG.md 2026-10-02).
+ */
+async function dropCopiedParts(binary: { id: number }) {
+  await db.from('parts').where('binary_id', binary.id).whereNull('pcrc32').delete()
 }

commands/workers/Spots.ts

@@ -6,6 +6,10 @@ import NntpService from '#services/NntpService'
 import SpotService from '#services/SpotService'
 
 const IDLE_MS = 30_000
+/** Spots claimed at a time; most of a history backfill is listed in bulk. */
+const BATCH = 500
+/** Imported spots whose details are read per idle round. */
+const DETAILS_BATCH = 50
 
 /**
  * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
@@ -29,17 +33,22 @@ export default class SpotsWorker extends BaseCommand {
     let done = 0
     try {
       while (!Shutdown.requested && (!this.limit || done < this.limit)) {
-        const batch = await SpotService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
         if (batch.length === 0) {
           if (this.limit) break
-          await Shutdown.sleep(IDLE_MS)
+          // Idle: read the details of imported spots that don't have them yet (newest first).
+          if ((await SpotService.fillDetails(DETAILS_BATCH)) === 0) await Shutdown.sleep(IDLE_MS)
           continue
         }
+        // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
+        const listable = batch.filter((spot) => SpotService.isListable(spot))
+        const single = batch.filter((spot) => !SpotService.isListable(spot))
+        counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
         let next = 0
         await Promise.all(
           Array.from({ length: concurrency }, async () => {
-            while (next < batch.length) {
-              const status = await SpotService.process(batch[next++])
+            while (next < single.length) {
+              const status = await SpotService.process(single[next++])
               counts[status] = (counts[status] ?? 0) + 1
             }
           })

config/gc.ts

@@ -5,7 +5,7 @@ import env from '#start/env'
  */
 export default {
   /** How often `gc:scheduler` sweeps. */
-  intervalMs: env.get('GC_INTERVAL_MS', 10 * 60 * 1000),
+  intervalMs: env.get('GC_INTERVAL_MS', 3 * 60 * 1000),
 
   jobs: {
     /** Completed jobs are only kept for inspection; keep them briefly. */
@@ -34,14 +34,34 @@ export default {
      * complete from headers. A real file gets its next articles in minutes.
      */
     singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60),
+    /**
+     * Completed header-derived binaries without a fingerprint (size, MD5 of
+     * the first 16 KiB): their file exists and no par2 set can name them
+     * later, so they go sooner than staleAfterSeconds.
+     */
+    unfingerprintedAfterSeconds: env.get('GC_UNFINGERPRINTED_SECONDS', 6 * 60 * 60),
   },
 
+  /**
+   * Staged lone segments (CONCEPT.md D35) are dropped after this long. A
+   * real file's next segment comes in the same or the next fetch range,
+   * seconds to minutes later; at backfill speed two hours of waiting
+   * segments were 12M rows, 10 GB.
+   */
+  stagingTtlSeconds: env.get('STAGING_TTL_SECONDS', 30 * 60),
+
   /** Releases without files or binaries (their binaries were collected) are deleted after this long. */
   emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60),
 
-  /** Bulk deletes run in batches of this size, for at most this long per sweep. */
+  /**
+   * Bulk deletes run in batches of this size, each kind (copied parts,
+   * single-article and unfingerprinted binaries, empty releases) for at most
+   * this long per sweep, so one backlog can't starve the others.
+   */
   deleteBatch: 20_000,
-  deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 3 * 60 * 1000),
+  deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 60 * 1000),
+  /** Bloated indexes are rebuilt at most this often (IndexMaintenance). */
+  reindexIntervalMs: env.get('GC_REINDEX_INTERVAL_MS', 7 * 24 * 60 * 60 * 1000),
 
   /**
    * Deferred files (CONCEPT.md D28) are deleted this long after they were

config/indexer.ts

@@ -39,6 +39,43 @@ export default {
    * binaries. Add one with `group:add free.pt --backfill 5000`.
    */
   spotGroups: ['free.pt'],
+  /**
+   * Spots posted within this many days are imported (their NZB read and its
+   * files added). Older spots are listed: title, size and date, with the NZB
+   * read on first download.
+   */
+  spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),
+
+  /**
+   * Groups whose lone header segments wait in `staged_segments` until their
+   * file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
+   */
+  stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
+
+  /**
+   * Groups whose staged segments also group by poster, file counter `[i/M]`
+   * and part count (CONCEPT.md D36), for uploads with a random subject per
+   * article. Needs staging. Comma-separated; `*` for all.
+   */
+  counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
+
+  /**
+   * Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
+   * header-only fetch ranges walking back through a group's history.
+   */
+  backfill: {
+    /** Backfill ranges waiting or running at a time, over all groups. */
+    maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
+    /** BullMQ priority (higher runs later; live ranges have none, so they go first). */
+    priority: 10,
+    /**
+     * No new backfill ranges while the database's NVMe part (default
+     * tablespace) or its HDD part (tablespace `hdd`, the files table) is
+     * larger than this. Checked every scheduler tick.
+     */
+    maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
+    maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
+  },
 
   /** worker:deferred leaves a file alone until it has had no new articles for this long. */
   deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),

config/nntp.ts

@@ -8,4 +8,6 @@ export default {
   secure: env.get('NNTP_SECURE', false),
   /** Connections per process. Mind the provider's limit across all processes. */
   poolSize: env.get('NNTP_POOL_SIZE', 10),
+  /** A command without an answer this long counts as a dead connection (XOVER of 20k headers takes ~0.2 s). */
+  commandTimeoutMs: env.get('NNTP_COMMAND_TIMEOUT_MS', 120_000),
 }

database/migrations/1768620765010_spot_listings.ts

@@ -22,11 +22,16 @@ export default class extends BaseSchema {
 
   async up() {
     await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`)
+    // worker:spots claims recent spots first, then the history in posting order.
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
     await this.swap(VISIBLE)
   }
 
   async down() {
     await this.swap('file_count > 0')
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_article_idx`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS spots_status_posted_idx`)
     await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`)
   }
 

database/migrations/1768620765020_spot_details.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Spot details (CONCEPT.md D29): the description and image from the spot's
+ * full header. Stored whenever the header is read (recent spots, moderation,
+ * downloads); for other old spots on first view. `details_at` marks the read.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('spots', (table) => {
+      table.text('description')
+      table.jsonb('image')
+      table.timestamp('details_at', { useTz: true })
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('spots', (table) => {
+      table.dropColumn('description')
+      table.dropColumn('image')
+      table.dropColumn('details_at')
+    })
+  }
+}

database/migrations/1768620765030_release_password_source.ts

@@ -0,0 +1,16 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Where a release's archive password came from: `nzb` (the NZB's
+ * `<meta type="password">`) or `spot` (the spot's title or description).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('releases', (table) => table.string('password_source', 8))
+    this.schema.raw(`UPDATE releases SET password_source = 'nzb' WHERE password IS NOT NULL`)
+  }
+
+  async down() {
+    this.schema.alterTable('releases', (table) => table.dropColumn('password_source'))
+  }
+}

database/migrations/1768620765040_pack_file_segments.ts

@@ -0,0 +1,63 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+import { packSegments, unpackSegments } from '#lib/nzb/segments'
+
+const BATCH = 5000
+
+/**
+ * Segment lists compressed (CONCEPT.md D31): `files.message_ids` (jsonb,
+ * ~38 bytes per segment after pglz) becomes `files.segments` (zstd, ~20),
+ * plus `first_message_id` for the lookups the old expression index served.
+ * Run with the workers stopped; reclaim the space with VACUUM FULL files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS segments bytea, ADD COLUMN IF NOT EXISTS first_message_id text`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await this.db.rawQuery(
+        `SELECT id, message_ids FROM files WHERE id > ? AND segments IS NULL ORDER BY id LIMIT ?`,
+        [last, BATCH]
+      )
+      if (rows.length === 0) break
+      const ids: number[] = []
+      const packed: Buffer[] = []
+      const firsts: (string | null)[] = []
+      for (const row of rows) {
+        const segments = Object.entries((row.message_ids ?? {}) as Record<string, { id: string; size?: number }>)
+          .map(([number, s]) => ({ number: Number(number), id: s.id, size: Number(s.size ?? 0) }))
+          .sort((a, b) => a.number - b.number)
+        ids.push(Number(row.id))
+        packed.push(packSegments(segments))
+        firsts.push(segments[0]?.id ?? null)
+      }
+      await this.db.rawQuery(
+        `UPDATE files f SET segments = v.segments, first_message_id = v.first
+         FROM unnest(?::bigint[], ?::bytea[], ?::text[]) AS v(id, segments, first) WHERE f.id = v.id`,
+        [ids, packed, firsts] as any
+      )
+      last = ids[ids.length - 1]
+      done += rows.length
+      if (done % 100_000 < BATCH) console.log(`Packed the segments of ${done} files.`)
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN segments SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_first_message_id_idx ON files (first_message_id)`)
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_first_segment_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN message_ids`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`ALTER TABLE files ADD COLUMN IF NOT EXISTS message_ids jsonb`)
+    const { rows } = await this.db.rawQuery(`SELECT id, segments FROM files`)
+    for (const row of rows) {
+      const json = Object.fromEntries(unpackSegments(row.segments).map((s) => [s.number, { id: s.id, size: s.size }]))
+      await this.db.rawQuery(`UPDATE files SET message_ids = ? WHERE id = ?`, [JSON.stringify(json), row.id])
+    }
+    await this.db.rawQuery(`ALTER TABLE files ALTER COLUMN message_ids SET NOT NULL`)
+    await this.db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`)
+    await this.db.rawQuery(`DROP INDEX IF EXISTS files_first_message_id_idx`)
+    await this.db.rawQuery(`ALTER TABLE files DROP COLUMN segments, DROP COLUMN first_message_id`)
+  }
+}

database/migrations/1768620765050_group_backfill.ts

@@ -0,0 +1,26 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Header backfill (CONCEPT.md D32): a group walks backwards from
+ * `backfill_next_id` down to `backfill_until_id` in low-priority,
+ * header-only fetch ranges (`fetch_ranges.backfill`).
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.alterTable('groups', (table) => {
+      table.bigInteger('backfill_until_id')
+      table.bigInteger('backfill_next_id')
+    })
+    this.schema.alterTable('fetch_ranges', (table) => {
+      table.boolean('backfill').notNullable().defaultTo(false)
+    })
+  }
+
+  async down() {
+    this.schema.alterTable('groups', (table) => {
+      table.dropColumn('backfill_until_id')
+      table.dropColumn('backfill_next_id')
+    })
+    this.schema.alterTable('fetch_ranges', (table) => table.dropColumn('backfill'))
+  }
+}

database/migrations/1768620765060_search_text_and_index_cleanup.ts

@@ -0,0 +1,144 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+const VISIBLE = `(file_count > 0 OR source = 'spot')`
+const BATCH = 20_000
+
+/**
+ * Search and index cleanup (CONCEPT.md D33).
+ * - `releases.search_text`: name, archive listing and readable file-name
+ *   stems, normalised (lowercase; dots, dashes, underscores and spaces as
+ *   one space), kept by triggers; one trigram index replaces the four the
+ *   old per-table OR search could hardly use.
+ * - Spot status indexes become partial indexes over the few rows the worker
+ *   and search look for.
+ * - Unused release indexes go (size: never scanned; complete: a boolean).
+ * Runs online: indexes are built and dropped concurrently.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
+    // For IndexMaintenance; needs a superuser (it is one in the compose setup).
+    await q(`CREATE EXTENSION IF NOT EXISTS pgstattuple`).catch((error) => console.warn(`pgstattuple not installed: ${error.message}`))
+
+    // Spots: the worker only looks for pending/processing and un-detailed imported spots, search for hidden ones.
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_todo_idx ON spots (article_number) WHERE status IN ('pending', 'processing')`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_hidden_release_idx ON spots (release_id) WHERE status IN ('removed', 'dummy')`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_details_todo_idx ON spots (posted_at DESC) WHERE status = 'imported' AND details_at IS NULL`)
+    for (const index of ['spots_status_posted_idx', 'spots_status_article_idx', 'spots_status_created_at_index', 'spots_status_release_id_index']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+    for (const index of ['releases_size_index', 'releases_complete_idx']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+
+    // Search text.
+    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS search_files text, ADD COLUMN IF NOT EXISTS search_text text`)
+    await q(`
+      CREATE OR REPLACE FUNCTION search_normalize(t text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT btrim(lower(regexp_replace(coalesce(t, ''), '[._[:space:]-]+', ' ', 'g')))
+      $$`)
+    await q(`
+      CREATE OR REPLACE FUNCTION release_search_text(name text, contents jsonb, files text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
+        SELECT left(concat_ws(' ',
+          search_normalize(name),
+          nullif(search_normalize((SELECT string_agg(e->>'name', ' ') FROM (
+            SELECT e FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) e LIMIT 200) x)), ''),
+          nullif(files, '')), 8000)
+      $$`)
+    // Visible releases only; computed when one becomes visible or its name, listing or file names change.
+    await q(`
+      CREATE OR REPLACE FUNCTION releases_search_text() RETURNS trigger LANGUAGE plpgsql AS $$
+      BEGIN
+        IF NOT (NEW.file_count > 0 OR NEW.source = 'spot') THEN
+          NEW.search_text := NULL;
+        ELSIF TG_OP = 'INSERT' OR NEW.search_text IS NULL
+          OR NEW.name IS DISTINCT FROM OLD.name OR NEW.contents IS DISTINCT FROM OLD.contents
+          OR NEW.search_files IS DISTINCT FROM OLD.search_files THEN
+          NEW.search_text := release_search_text(NEW.name, NEW.contents, NEW.search_files);
+        END IF;
+        RETURN NEW;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
+    await q(`CREATE TRIGGER releases_search_text BEFORE INSERT OR UPDATE ON releases FOR EACH ROW EXECUTE FUNCTION releases_search_text()`)
+    // A file's readable stem joins its release's search text when it adds something.
+    await q(`
+      CREATE OR REPLACE FUNCTION files_release_search() RETURNS trigger LANGUAGE plpgsql AS $$
+      DECLARE
+        n text := coalesce(NEW.real_name, NEW.filename);
+        s text;
+      BEGIN
+        IF NEW.release_id IS NULL OR NOT is_readable_name(n) THEN RETURN NULL; END IF;
+        s := search_normalize(name_stem(n));
+        IF s = '' THEN RETURN NULL; END IF;
+        UPDATE releases SET search_files = left(concat_ws(' ', search_files, s), 4000)
+        WHERE id = NEW.release_id AND position(s IN coalesce(search_text, search_normalize(name))) = 0
+          AND length(coalesce(search_files, '')) < 4000;
+        RETURN NULL;
+      END $$`)
+    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
+    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)
+
+    // Fill: readable file stems per release (one pass over files), then every visible release in batches.
+    await q(`DROP TABLE IF EXISTS search_stems_tmp`)
+    await q(`
+      CREATE UNLOGGED TABLE search_stems_tmp AS
+      SELECT release_id, left(string_agg(DISTINCT s, ' '), 4000) AS stems FROM (
+        SELECT f.release_id, search_normalize(name_stem(coalesce(f.real_name, f.filename))) AS s
+        FROM files f WHERE f.release_id IS NOT NULL AND is_readable_name(coalesce(f.real_name, f.filename))) x
+      WHERE s <> '' GROUP BY release_id`)
+    await q(`CREATE UNIQUE INDEX ON search_stems_tmp (release_id)`)
+    let last = 0
+    let done = 0
+    while (true) {
+      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
+      if (rows.length === 0) break
+      const to = Number(rows[rows.length - 1].id)
+      await q(
+        `UPDATE releases r SET search_files = (
+           SELECT nullif(array_to_string(array(
+             SELECT w FROM unnest(string_to_array(t.stems, ' ')) w
+             WHERE position(w IN search_normalize(r.name)) = 0), ' '), '')
+           FROM search_stems_tmp t WHERE t.release_id = r.id),
+           search_text = NULL
+         WHERE r.id > ? AND r.id <= ? AND ${VISIBLE.replaceAll('file_count', 'r.file_count').replace('source', 'r.source')}`,
+        [last, to]
+      )
+      last = to
+      done += rows.length
+      if (done % 200_000 < BATCH) console.log(`Search text for ${done} releases.`)
+    }
+    await q(`DROP TABLE search_stems_tmp`)
+
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_search_trgm ON releases USING gin (search_text gin_trgm_ops) WHERE ${VISIBLE}`)
+    for (const index of ['releases_name_trgm', 'releases_contents_trgm', 'files_filename_trgm', 'files_real_name_trgm']) {
+      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
+    }
+  }
+
+  async down() {
+    const q = (sql: string) => this.db.rawQuery(sql)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_search_trgm`)
+    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
+    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
+    await q(`DROP FUNCTION IF EXISTS files_release_search()`)
+    await q(`DROP FUNCTION IF EXISTS releases_search_text()`)
+    await q(`DROP FUNCTION IF EXISTS release_search_text(text, jsonb, text)`)
+    await q(`DROP FUNCTION IF EXISTS search_normalize(text)`)
+    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS search_files, DROP COLUMN IF EXISTS search_text`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_size_index ON releases (size) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_complete_idx ON releases (complete) WHERE ${VISIBLE}`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_created_at_index ON spots (status, created_at)`)
+    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_release_id_index ON spots (status, release_id)`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_todo_idx`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_hidden_release_idx`)
+    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_details_todo_idx`)
+  }
+}

database/migrations/1768620765070_readable_name_more_shapes.ts

@@ -0,0 +1,43 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * is_readable_name(): also hex with a Unix time, UUIDs, digit-only names
+ * (and rclone's "83111763 n"), lowercase base32/36 tokens and short tokens
+ * with capitals all over. The same rules as looksRandom() in ArchiveService.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      await db.rawQuery(`
+        CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+          WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+          SELECT name IS NOT NULL AND NOT (
+            (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+              AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+            OR stem ~* '^[0-9a-f]{16,}$'
+            OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
+            OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+            OR stem ~ '^\\d{6,}( n){0,1}$'
+            OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
+                AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
+            OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
+                AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3))
+          FROM s
+        $$`)
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db.rawQuery(`
+        CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+          WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+          SELECT name IS NOT NULL AND NOT (
+            (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+              AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+            OR stem ~* '^[0-9a-f]{16,}$')
+          FROM s
+        $$`)
+    })
+  }
+}

database/migrations/1768620765080_create_sfv_entries.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * SFV listings (CONCEPT.md D34): the files an SFV names belong to its
+ * release; files arriving later are matched by name.
+ */
+export default class extends BaseSchema {
+  async up() {
+    this.schema.createTable('sfv_entries', (table) => {
+      table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
+      table.text('name').notNullable()
+      table.string('crc', 8).notNullable()
+      table.text('poster')
+      table.timestamp('posted_at', { useTz: true })
+      table.primary(['release_id', 'name'])
+      table.index(['name'])
+    })
+  }
+
+  async down() {
+    this.schema.dropTable('sfv_entries')
+  }
+}

database/migrations/1768620765090_files_name_index.ts

@@ -0,0 +1,15 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/** SFV grouping (CONCEPT.md D34) looks files up by their name. */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    // On the NVMe (default tablespace), like the files table's other indexes.
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_name_idx ON files ((coalesce(real_name, filename)))`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS files_name_idx`)
+  }
+}

database/migrations/1768620765100_readable_name_generic.ts

@@ -0,0 +1,37 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * is_readable_name(): generic folder/file names (VIDEO_TS, Sample, CD1) are
+ * no name either. Same list as GENERIC_NAMES in lib/release/release_key.ts.
+ */
+const FUNCTION = (generic: boolean) => `
+  CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$
+    WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem)
+    SELECT name IS NOT NULL AND NOT (
+      (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]'
+        AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6))
+      OR stem ~* '^[0-9a-f]{16,}$'
+      OR stem ~* '^[0-9a-f]{16,}\\.\\d{9,}$'
+      OR stem ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+      OR stem ~ '^\\d{6,}( n){0,1}$'
+      OR (stem ~ '^[a-z0-9]{12,}$' AND length(regexp_replace(stem, '\\D', '', 'g')) >= 2
+          AND length(regexp_replace(stem, '[^a-z]', '', 'g')) >= 4)
+      OR (stem ~ '^[A-Za-z0-9]{8,15}$' AND stem ~ '\\d' AND stem ~ '[a-z]'
+          AND length(regexp_replace(substr(stem, 2), '[^A-Z]', '', 'g')) >= 3)
+      ${generic ? `OR btrim(stem) ~* '^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples{0,1}|subs{0,1}|subtitles{0,1}|proof|cover|covers|extras{0,1}|featurettes{0,1}|(cd|dvd|disc|disk)[ ._-]{0,1}\\d{1,2})$'` : ''})
+    FROM s
+  $$`
+
+export default class extends BaseSchema {
+  async up() {
+    this.defer(async (db) => {
+      await db.rawQuery(FUNCTION(true))
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db.rawQuery(FUNCTION(false))
+    })
+  }
+}

database/migrations/1768620765110_create_staged_segments.ts

@@ -0,0 +1,24 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Staged segments (CONCEPT.md D35): a header segment whose file is new and
+ * alone in its batch waits here until a second segment of the file shows
+ * up; ~97% never do (uploads with a random subject per article). Unlogged:
+ * no WAL, emptied by a crash, which only costs those waiting segments.
+ */
+export default class extends BaseSchema {
+  async up() {
+    await this.db.rawQuery(`
+      CREATE UNLOGGED TABLE staged_segments (
+        file_key text PRIMARY KEY,
+        segments jsonb NOT NULL,
+        staged_at timestamptz NOT NULL
+      )`)
+    // Rows arrive in time order, so a BRIN index serves the expiry cheaply.
+    await this.db.rawQuery(`CREATE INDEX staged_segments_staged_at_brin ON staged_segments USING brin (staged_at)`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP TABLE staged_segments`)
+  }
+}

database/migrations/1768620765120_staged_segments_counter_key.ts

@@ -0,0 +1,20 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+/**
+ * Counter grouping (CONCEPT.md D36): staged segments also carry the key of
+ * the file they'd belong to by poster, file counter `[i/M]`, part count and
+ * posting day, so uploads with a random subject per article become files.
+ */
+export default class extends BaseSchema {
+  static disableTransactions = true
+
+  async up() {
+    await this.db.rawQuery(`ALTER TABLE staged_segments ADD COLUMN IF NOT EXISTS counter_key text`)
+    await this.db.rawQuery(`CREATE INDEX CONCURRENTLY IF NOT EXISTS staged_segments_counter_key_idx ON staged_segments (counter_key) WHERE counter_key IS NOT NULL`)
+  }
+
+  async down() {
+    await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS staged_segments_counter_key_idx`)
+    await this.db.rawQuery(`ALTER TABLE staged_segments DROP COLUMN IF EXISTS counter_key`)
+  }
+}

database/schema.ts

@@ -148,8 +148,10 @@ export class DeferredRangeSchema extends BaseModel {
 }
 
 export class FetchRangeSchema extends BaseModel {
-  static $columns = ['createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
+  static $columns = ['backfill', 'createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const
   $columns = FetchRangeSchema.$columns
+  @column()
+  declare backfill: boolean
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
   @column()
@@ -171,7 +173,7 @@ export class FetchRangeSchema extends BaseModel {
 }
 
 export class FileSchema extends BaseModel {
-  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'realName', 'releaseId', 'size', 'updatedAt'] as const
+  static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'firstMessageId', 'groups', 'id', 'parts', 'poster', 'realName', 'releaseId', 'segments', 'size', 'updatedAt'] as const
   $columns = FileSchema.$columns
   @column()
   declare archive: any | null
@@ -186,12 +188,12 @@ export class FileSchema extends BaseModel {
   @column()
   declare filename: string
   @column()
+  declare firstMessageId: string | null
+  @column()
   declare groups: any | null
   @column({ isPrimary: true })
   declare id: number
   @column()
-  declare messageIds: any
-  @column()
   declare parts: number
   @column()
   declare poster: string
@@ -200,16 +202,22 @@ export class FileSchema extends BaseModel {
   @column()
   declare releaseId: bigint | number | null
   @column()
+  declare segments: Buffer
+  @column()
   declare size: bigint | number | null
   @column.dateTime({ autoCreate: true, autoUpdate: true })
   declare updatedAt: DateTime | null
 }
 
 export class GroupSchema extends BaseModel {
-  static $columns = ['active', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
+  static $columns = ['active', 'backfillNextId', 'backfillUntilId', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const
   $columns = GroupSchema.$columns
   @column()
   declare active: boolean
+  @column()
+  declare backfillNextId: bigint | number | null
+  @column()
+  declare backfillUntilId: bigint | number | null
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime | null
   @column()
@@ -292,7 +300,7 @@ export class ReleaseFileSchema extends BaseModel {
 }
 
 export class ReleaseSchema extends BaseModel {
-  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'postedAt', 'poster', 'releaseKey', 'size', 'source'] as const
+  static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'passwordSource', 'postedAt', 'poster', 'releaseKey', 'searchFiles', 'searchText', 'size', 'source'] as const
   $columns = ReleaseSchema.$columns
   @column()
   declare category: number | null
@@ -320,6 +328,8 @@ export class ReleaseSchema extends BaseModel {
   declare name: string
   @column({ serializeAs: null })
   declare password: string | null
+  @column()
+  declare passwordSource: string | null
   @column.dateTime()
   declare postedAt: DateTime | null
   @column()
@@ -327,13 +337,32 @@ export class ReleaseSchema extends BaseModel {
   @column()
   declare releaseKey: string
   @column()
+  declare searchFiles: string | null
+  @column()
+  declare searchText: string | null
+  @column()
   declare size: bigint | number
   @column()
   declare source: string | null
 }
 
+export class SfvEntrySchema extends BaseModel {
+  static $columns = ['crc', 'name', 'postedAt', 'poster', 'releaseId'] as const
+  $columns = SfvEntrySchema.$columns
+  @column()
+  declare crc: string
+  @column()
+  declare name: string
+  @column.dateTime()
+  declare postedAt: DateTime | null
+  @column()
+  declare poster: string | null
+  @column({ isPrimary: true })
+  declare releaseId: bigint | number
+}
+
 export class SpotSchema extends BaseModel {
-  static $columns = ['articleNumber', 'category', 'createdAt', 'error', 'groupName', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'website'] as const
+  static $columns = ['articleNumber', 'category', 'createdAt', 'description', 'detailsAt', 'error', 'groupName', 'image', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'verified', 'website'] as const
   $columns = SpotSchema.$columns
   @column()
   declare articleNumber: bigint | number | null
@@ -342,10 +371,16 @@ export class SpotSchema extends BaseModel {
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
   @column()
+  declare description: string | null
+  @column.dateTime()
+  declare detailsAt: DateTime | null
+  @column()
   declare error: string | null
   @column()
   declare groupName: string
   @column()
+  declare image: any | null
+  @column()
   declare keyId: number | null
   @column({ isPrimary: true })
   declare messageId: string
@@ -370,9 +405,24 @@ export class SpotSchema extends BaseModel {
   @column()
   declare title: string
   @column()
+  declare verified: boolean | null
+  @column()
   declare website: string | null
 }
 
+export class StagedSegmentSchema extends BaseModel {
+  static $columns = ['counterKey', 'fileKey', 'segments', 'stagedAt'] as const
+  $columns = StagedSegmentSchema.$columns
+  @column()
+  declare counterKey: string | null
+  @column({ isPrimary: true })
+  declare fileKey: string
+  @column()
+  declare segments: any
+  @column.dateTime()
+  declare stagedAt: DateTime
+}
+
 export class UserSchema extends BaseModel {
   static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
   $columns = UserSchema.$columns

lib/content/archive_names.ts

@@ -0,0 +1,13 @@
+import { sevenZipVolume } from '../sevenzip/index.js'
+
+/**
+ * Files whose first segment says what an archive holds: named first RAR
+ * volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
+ * uploaders use for archive volumes too (sniffed by content).
+ */
+export function isArchiveStart(name: string): boolean {
+  if (/\.bin$/i.test(name)) return true
+  const volume = sevenZipVolume(name)
+  if (volume === 0 || volume === 1) return true
+  return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
+}

lib/content/index.ts

@@ -2,3 +2,6 @@
  * Content recognition by magic bytes, independent of AdonisJS.
  */
 export { sniffContentType, type ContentType } from './sniff.js'
+export { isArchiveStart } from './archive_names.js'
+export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js'
+export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js'

lib/content/nicovideo.ts

@@ -0,0 +1,76 @@
+import { createZstdDecompress } from 'node:zlib'
+
+/**
+ * Captures of niconico videos (`<unixtime>_nicovideo_jp_watch_sm<id>.tar.zst`,
+ * one poster, ~190k files on homelab). The tar's first entry after its
+ * directory is `server_response.metadata`: the site's API response for the
+ * watch page, 30–70 KB, so the first article of the file holds it.
+ */
+export const NICOVIDEO_FILE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)\.tar\.zst$/i
+export const NICOVIDEO_RELEASE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)$/i
+
+export interface NicovideoInfo {
+  id: string
+  title: string
+  owner: string | null
+  registeredAt: string | null
+  tags: string[]
+}
+
+/** As much of a zstd stream as `data` (a prefix of it) decompresses to. */
+export function zstdPrefix(data: Buffer, maxBytes = 4 * 1024 * 1024): Promise<Buffer> {
+  return new Promise((resolve) => {
+    const out: Buffer[] = []
+    let length = 0
+    const z = createZstdDecompress()
+    const done = () => resolve(Buffer.concat(out))
+    z.on('data', (chunk: Buffer) => {
+      out.push(chunk)
+      length += chunk.length
+      if (length >= maxBytes) z.destroy()
+    })
+    z.on('error', done)
+    z.on('close', done)
+    z.on('end', done)
+    z.end(data)
+  })
+}
+
+/** Entries at the start of a (possibly truncated) ustar archive. */
+export function* tarEntries(tar: Buffer): Generator<{ name: string; data: Buffer }> {
+  let offset = 0
+  while (offset + 512 <= tar.length) {
+    const header = tar.subarray(offset, offset + 512)
+    const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '')
+    if (!name) return
+    const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, '')
+    const size = Number.parseInt(header.subarray(124, 136).toString('latin1').replace(/\0.*$/s, '').trim() || '0', 8) || 0
+    yield { name: prefix ? `${prefix}/${name}` : name, data: tar.subarray(offset + 512, offset + 512 + size) }
+    offset += 512 + Math.ceil(size / 512) * 512
+  }
+}
+
+/** Title, uploader and tags from the start of a capture, or null when they aren't in it. */
+export function nicovideoInfo(tar: Buffer): NicovideoInfo | null {
+  for (const entry of tarEntries(tar)) {
+    if (!entry.name.endsWith('/server_response.metadata')) continue
+    try {
+      const text = entry.data.toString('utf8')
+      const json = JSON.parse(text.slice(text.indexOf('{')))
+      const video = json?.data?.response?.video
+      if (!video?.title) return null
+      const owner = json.data.response.owner?.nickname ?? json.data.response.channel?.name ?? json.data.metadata?.jsonLds?.[0]?.author?.name ?? null
+      const tags = (json.data.response.tag?.items ?? []).map((t: any) => String(t.name)).filter(Boolean)
+      return { id: String(video.id ?? ''), title: String(video.title).trim(), owner: owner ? String(owner).trim() : null, registeredAt: video.registeredAt ?? null, tags }
+    } catch {
+      return null
+    }
+  }
+  return null
+}
+
+/** Release name for a capture: `<title> - <uploader> (nicovideo sm123)`. */
+export function nicovideoReleaseName(info: NicovideoInfo, fallbackId: string): string {
+  const id = info.id || fallbackId
+  return `${info.title}${info.owner ? ` - ${info.owner}` : ''} (nicovideo ${id})`
+}

lib/content/sfv.ts

@@ -0,0 +1,29 @@
+/**
+ * SFV checksum lists: one `name CRC32` per line, `;` comments. Scene
+ * releases (music in particular) post one with every album; it names every
+ * file that belongs to it.
+ */
+export interface SfvEntry {
+  name: string
+  crc: string
+}
+
+export function parseSfv(text: string): SfvEntry[] {
+  const entries: SfvEntry[] = []
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.trim()
+    if (!line || line.startsWith(';')) continue
+    const match = /^(.+?)\s+([0-9a-f]{8})$/i.exec(line)
+    if (match) entries.push({ name: match[1].replace(/^"|"$/g, '').split(/[\\/]/).pop()!, crc: match[2].toLowerCase() })
+  }
+  return entries
+}
+
+/**
+ * The album name an SFV gives a release: its stem without the leading
+ * track number scene groups put on the playlist/SFV/cover files
+ * (`00-artist-album-web-2026-grp.sfv` → `artist-album-web-2026-grp`).
+ */
+export function sfvReleaseName(sfvName: string): string {
+  return sfvName.replace(/\.sfv$/i, '').replace(/^0{1,3}-/, '')
+}

lib/content/sniff.ts

@@ -2,6 +2,8 @@
  * Recognises a file type from its first bytes, independent of AdonisJS.
  * `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
  * "nzbe", 16 random bytes (salt/IV), then ciphertext.
+ * `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com`
+ * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
  */
 export type ContentType =
   | 'nzbe'
@@ -16,6 +18,7 @@ export type ContentType =
   | 'matroska'
   | 'mp4'
   | 'rclone'
+  | 'yencbin'
   | 'iso'
   | 'udf'
   | 'pdf'
@@ -29,7 +32,11 @@ export type ContentType =
   | 'ogg'
   | 'bzip2'
 
-const SIGNATURES: [ContentType, number, number[]][] = [
+/** Magic bytes at an offset; null matches any byte. */
+const SIGNATURES: [ContentType, number, (number | null)[]][] = [
+  // Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…):
+  // the bytes they share.
+  ['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]],
   ['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
   ['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
   ['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
@@ -53,7 +60,7 @@ const SIGNATURES: [ContentType, number, number[]][] = [
 
 export function sniffContentType(data: Buffer): ContentType | null {
   for (const [type, offset, bytes] of SIGNATURES) {
-    if (data.length >= offset + bytes.length && bytes.every((b, i) => data[offset + i] === b)) return type
+    if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type
   }
   // Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
   // ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.

lib/newznab/xml.ts

@@ -13,6 +13,8 @@ export interface NewznabItem {
   poster: string | null
   groups: string[]
   password: boolean
+  /** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */
+  description?: string
   season?: string
   episode?: string
 }
@@ -26,6 +28,16 @@ export interface NewznabServer {
 }
 
 /** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
+/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
+export function ubbToText(ubb: string): string {
+  return ubb
+    .replace(/<br\s*\/?>|\[br\]/gi, '\n')
+    .replace(/<\/?[a-z][^>]*>/gi, '')
+    .replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim()))
+    .replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '')
+    .trim()
+}
+
 export function errorXml(code: number, description: string): string {
   return create({ version: '1.0', encoding: 'UTF-8' })
     .ele('error', { code: String(code), description })
@@ -74,7 +86,7 @@ export function resultsXml(server: NewznabServer, items: NewznabItem[], offset:
     element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
     element.ele('pubDate').txt(pubDate)
     element.ele('category').txt(categoryLabel(item.category))
-    element.ele('description').txt(item.title)
+    element.ele('description').txt(item.description ? ubbToText(item.description) : item.title)
     element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
 
     const attr = (name: string, value: string | number) =>

lib/nntp/date_search.ts

@@ -0,0 +1,27 @@
+/**
+ * The article number where a group's posts reach `target` (ms since epoch),
+ * by binary search between `first` and `last`. Article numbers only roughly
+ * follow posting time, so the answer is within `precision` articles of the
+ * first one posted at or after `target`.
+ *
+ * `dateOf(n)` returns the posting time of the first article at or after `n`
+ * that still exists, or null when there is none nearby (expired or removed);
+ * a gap counts as older than `target`.
+ */
+export async function articleAtDate(
+  first: bigint,
+  last: bigint,
+  target: number,
+  dateOf: (n: bigint) => Promise<number | null>,
+  precision = 10_000n
+): Promise<bigint> {
+  let lo = first
+  let hi = last
+  while (hi - lo > precision) {
+    const mid = lo + (hi - lo) / 2n
+    const date = await dateOf(mid)
+    if (date === null || date < target) lo = mid
+    else hi = mid
+  }
+  return lo
+}

lib/nntp/delimited_reader.ts

@@ -0,0 +1,76 @@
+const CRLF = Buffer.from('\r\n')
+
+/**
+ * Splits a byte stream into messages that end in a delimiter, which the
+ * consumer can change per message: `next(delimiter)` returns everything up
+ * to the next `delimiter` (excluded). A replacement for nntp-js's
+ * findByteSequence, which has two bugs:
+ *
+ * - An empty multi-line reply (`224 …\r\n.\r\n`, e.g. XOVER of a range
+ *   without articles) never ends: the status line takes the CRLF that
+ *   `\r\n.\r\n` starts with. Here, when a delimiter that starts with CRLF
+ *   follows one that ended with CRLF, that CRLF counts for both.
+ * - A delimiter split over two socket chunks left its first bytes in the
+ *   returned data.
+ *
+ * Empty messages for a bare CRLF delimiter are skipped, as nntp-js does.
+ */
+export async function* delimitedReader(
+  source: AsyncIterable<Uint8Array>,
+  initial: Uint8Array
+): AsyncGenerator<Buffer, void, Uint8Array | undefined> {
+  let delimiter = Buffer.from(initial)
+  let parts: Buffer[] = []
+  let length = 0
+  // Bytes at the start of the buffer, already searched without a match.
+  let searched = 0
+  // Leading bytes that belong to the previous delimiter (a shared CRLF).
+  let skip = 0
+
+  for await (const chunk of source) {
+    parts.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))
+    length += chunk.byteLength
+
+    while (true) {
+      const from = Math.max(0, searched - (delimiter.length - 1))
+      const found = indexOf(parts, length, delimiter, from)
+      if (found === -1) {
+        searched = length
+        break
+      }
+      const all = parts.length === 1 ? parts[0] : Buffer.concat(parts, length)
+      const message = all.subarray(Math.min(skip, found), found)
+      const rest = all.subarray(found + delimiter.length)
+      const previous = delimiter
+
+      if (message.length === 0 && delimiter.equals(CRLF)) {
+        // A stray empty line: keep looking for the same delimiter.
+      } else {
+        delimiter = Buffer.from((yield message) ?? delimiter)
+      }
+
+      const shareCrlf = endsWithCrlf(previous) && delimiter.length > 2 && startsWithCrlf(delimiter)
+      parts = shareCrlf ? [CRLF, rest] : [rest]
+      length = rest.length + (shareCrlf ? 2 : 0)
+      skip = shareCrlf ? 2 : 0
+      searched = 0
+    }
+  }
+
+  if (length > skip) yield Buffer.concat(parts, length).subarray(skip)
+}
+
+const endsWithCrlf = (b: Buffer) => b.length >= 2 && b[b.length - 2] === 13 && b[b.length - 1] === 10
+const startsWithCrlf = (b: Buffer) => b.length >= 2 && b[0] === 13 && b[1] === 10
+
+/** indexOf over a list of buffers (their total `length`), starting at `from`; copies only the searched tail. */
+function indexOf(parts: Buffer[], length: number, needle: Buffer, from: number): number {
+  if (parts.length === 1) return parts[0].indexOf(needle, from)
+  // Collect the parts that cover [from, length).
+  let start = length
+  let i = parts.length
+  while (i > 0 && start > from) start -= parts[--i].length
+  const window = Buffer.concat(parts.slice(i), length - start)
+  const found = window.indexOf(needle, from - start)
+  return found === -1 ? -1 : start + found
+}

lib/nntp/nntp_js.ts

@@ -0,0 +1,39 @@
+import { NNTP } from 'nntp-js'
+import { delimitedReader } from './delimited_reader.js'
+
+const CRLF = new Uint8Array([13, 10])
+
+/**
+ * nntp-js with its reply reader replaced by `delimitedReader` (see there for
+ * the bugs it fixes; the empty-reply one hung XOVER of empty ranges forever).
+ * The reader is created in `connect`, so that is overridden: the same steps
+ * as nntp-js 1.0.4, with the other reader.
+ */
+const proto = NNTP.prototype as any
+if (!proto.__delimitedReader) {
+  proto.__delimitedReader = true
+  proto.connect = async function (this: any, readermode = false, timeout: number | undefined = undefined, socket?: any) {
+    try {
+      this.sock = socket ?? this._createSocket(timeout)
+      // The first reply is always the one-line welcome.
+      this.byteReader = delimitedReader(this.sock, CRLF)
+      this.welcome = new TextDecoder('utf-8').decode(await this._getresp())
+      this.caps = undefined
+      this.connected = true
+      await this.getcapabilities()
+      this.readermodeAfterauth = false
+      if (readermode && !this.caps.READER) {
+        await this._setreadermode()
+        if (!this.readermodeAfterauth) {
+          this.caps = undefined
+          await this.getcapabilities()
+        }
+      }
+    } catch (error) {
+      this.sock?.end()
+      throw error
+    }
+  }
+}
+
+export { NNTP }

lib/nzb/build.ts

@@ -1,4 +1,5 @@
 import { create } from 'xmlbuilder2'
+import type { Segment } from './segments.js'
 
 export interface NzbFile {
   /** Name the downloader should use; the real name when it is known. */
@@ -7,8 +8,7 @@ export interface NzbFile {
   /** Posting time in milliseconds. */
   date: number
   groups: string[]
-  /** Segment number → message-id (without angle brackets) and article size. */
-  segments: Record<string, { id: string; size: number }>
+  segments: Segment[]
 }
 
 export interface NzbOptions {
@@ -34,20 +34,17 @@ export function buildNzb(files: NzbFile[], options: NzbOptions = {}): string {
   }
 
   for (const file of files) {
-    const numbers = Object.keys(file.segments)
-      .map(Number)
-      .sort((a, b) => a - b)
+    const sorted = [...file.segments].sort((a, b) => a.number - b.number)
     const element = root.ele('file', {
       poster: file.poster,
       date: String(Math.floor(file.date / 1000)),
-      subject: `"${file.name}" yEnc (1/${numbers.length})`,
+      subject: `"${file.name}" yEnc (1/${sorted.length})`,
     })
     const groups = element.ele('groups')
     for (const group of file.groups) groups.ele('group').txt(group)
     const segments = element.ele('segments')
-    for (const number of numbers) {
-      const segment = file.segments[String(number)]
-      segments.ele('segment', { bytes: String(segment.size), number: String(number) }).txt(segment.id)
+    for (const segment of sorted) {
+      segments.ele('segment', { bytes: String(segment.size), number: String(segment.number) }).txt(segment.id)
     }
   }
   return root.end({ prettyPrint: true })

lib/nzb/index.ts

@@ -3,3 +3,4 @@
  */
 export { buildNzb, type NzbFile, type NzbOptions } from './build.js'
 export { parseNzb, type ParsedNzb, type ParsedNzbFile } from './parse.js'
+export type { Segment } from './segments.js'

lib/nzb/segments.ts

@@ -0,0 +1,33 @@
+import { zstdCompressSync, zstdDecompressSync, constants } from 'node:zlib'
+
+/** One article of a posted file: its number, message-id (without <>) and size in bytes. */
+export interface Segment {
+  number: number
+  id: string
+  size: number
+}
+
+const VERSION = 1
+
+/**
+ * Segment lists as stored in `files.segments` (CONCEPT.md D31): a version
+ * byte, then zstd (level 1) of a JSON line with the sizes (and the numbers,
+ * unless they are 1..N) followed by one message-id per line. About 20 bytes
+ * per segment, against ~38 for the pglz-compressed JSON object it replaces:
+ * message-ids are mostly random, so this is close to what's possible.
+ */
+export function packSegments(segments: Segment[]): Buffer {
+  const sorted = [...segments].sort((a, b) => a.number - b.number)
+  const sequential = sorted.every((s, i) => s.number === i + 1)
+  const head = JSON.stringify(sequential ? { s: sorted.map((s) => s.size) } : { s: sorted.map((s) => s.size), n: sorted.map((s) => s.number) })
+  const text = Buffer.from([head, ...sorted.map((s) => s.id)].join('\n'), 'utf8')
+  const packed = zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 1 } })
+  return Buffer.concat([Buffer.from([VERSION]), packed])
+}
+
+export function unpackSegments(data: Buffer): Segment[] {
+  if (data[0] !== VERSION) throw new Error(`Unknown segment list version ${data[0]}`)
+  const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
+  const { s: sizes, n: numbers } = JSON.parse(head) as { s: number[]; n?: number[] }
+  return sizes.map((size, i) => ({ number: numbers?.[i] ?? i + 1, id: ids[i], size }))
+}

lib/rar/parse.ts

@@ -19,7 +19,8 @@ export interface RarEntry {
 }
 
 export interface RarInfo {
-  format: 'rar4' | 'rar5'
+  /** 7z listings take the same shape (see lib/sevenzip). */
+  format: 'rar4' | 'rar5' | '7z'
   /** Part of a multi-volume set. */
   volume: boolean
   /** First volume of the set (null when the format version does not say). */

lib/release/index.ts

@@ -1,4 +1,4 @@
 /**
  * Grouping of files into releases (downloads), independent of AdonisJS.
  */
-export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'
+export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'

lib/release/release_key.ts

@@ -197,11 +197,23 @@ export function par2ReleaseName(fileNames: string[]): string | undefined {
  * volumes after the first, the first entry is the file that continues from
  * the previous volume, so all volumes of a single-file archive agree.
  */
+/**
+ * Folder and file names that say nothing about what a release is: disc
+ * structures (VIDEO_TS on every DVD), samples, subtitles, disc numbers.
+ * Kept in step with SQL's is_readable_name().
+ */
+export const GENERIC_NAMES = /^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?\d{1,2})$/i
+
+export function isGenericName(name: string): boolean {
+  return GENERIC_NAMES.test(name.trim())
+}
+
 export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
   if (entries.length === 0) return null
   const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
   const nested = entries.some((e) => /[\\/]/.test(e.name))
-  if (nested && tops.size === 1) {
+  // A generic top folder (VIDEO_TS) is shared by unrelated archives: key on the first entry instead.
+  if (nested && tops.size === 1 && !isGenericName([...tops][0])) {
     const [top] = tops
     return { key: hash('rar-dir', top), name: top, source: 'rar' }
   }

lib/sevenzip/index.ts

@@ -0,0 +1,97 @@
+import { crc32 } from 'node:zlib'
+
+/**
+ * 7-Zip archives, independent of AdonisJS. A 7z archive starts with a
+ * 32-byte signature header that points to the archive's header (the file
+ * list) at its very end, often LZMA-compressed ("encoded") with its packed
+ * data just before it. So the list needs the first 32 bytes and the tail:
+ * for a split archive (.7z.001, .002, …), the start of the first volume and
+ * the end of the last. Listing itself is left to libarchive (bsdtar).
+ */
+
+const SIGNATURE = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])
+const SIGNATURE_HEADER_SIZE = 32
+/** Header property ids (7zFormat.txt). */
+const K_ENCODED_HEADER = 0x17
+const K_PACK_INFO = 0x06
+
+export interface SevenZipStart {
+  /** Where the archive header starts, from the start of the archive. */
+  headerStart: number
+  headerSize: number
+  headerCrc: number
+  /** The archive's total size (the header is its last part). */
+  archiveSize: number
+}
+
+/** Reads the signature header; null when `data` isn't a 7z start or its CRC fails. */
+export function parseSevenZipStart(data: Buffer): SevenZipStart | null {
+  if (data.length < SIGNATURE_HEADER_SIZE || !data.subarray(0, 6).equals(SIGNATURE)) return null
+  if ((crc32(data.subarray(12, 32)) >>> 0) !== data.readUInt32LE(8)) return null
+  const offset = Number(data.readBigUInt64LE(12))
+  const size = Number(data.readBigUInt64LE(20))
+  const headerStart = SIGNATURE_HEADER_SIZE + offset
+  return { headerStart, headerSize: size, headerCrc: data.readUInt32LE(28), archiveSize: headerStart + size }
+}
+
+/** Whether `header` (the bytes at headerStart) is the archive header its signature header describes. */
+export function checkHeader(start: SevenZipStart, header: Buffer): boolean {
+  return header.length === start.headerSize && (crc32(header) >>> 0) === start.headerCrc
+}
+
+/**
+ * Where the bytes needed for listing start: the header itself, or for an
+ * encoded header the packed (compressed) header stream it points to.
+ */
+export function listingStart(start: SevenZipStart, header: Buffer): number {
+  if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return start.headerStart
+  const [packPos] = readNumber(header, 2)
+  return SIGNATURE_HEADER_SIZE + packPos
+}
+
+/**
+ * 7z's variable-length number: the leading 1-bits of the first byte count
+ * the little-endian bytes that follow; its remaining bits are the high part.
+ */
+export function readNumber(data: Buffer, pos: number): [value: number, next: number] {
+  const first = data[pos]
+  let mask = 0x80
+  let value = 0
+  for (let i = 0; i < 8; i++) {
+    if ((first & mask) === 0) {
+      value += (first & (mask - 1)) * 2 ** (8 * i)
+      return [value, pos + 1 + i]
+    }
+    value += data[pos + 1 + i] * 2 ** (8 * i)
+    mask >>= 1
+  }
+  return [value, pos + 9]
+}
+
+export interface SevenZipEntry {
+  name: string
+  size: number
+  directory: boolean
+}
+
+/**
+ * Entries from `bsdtar -tv` output, e.g.
+ * `-rw-rw-r--  0 0      0     3000000 Sep 30 19:13 Some.Release/movie.mkv`.
+ */
+export function parseBsdtarListing(stdout: string): SevenZipEntry[] {
+  const entries: SevenZipEntry[] = []
+  for (const line of stdout.split('\n')) {
+    const match = /^([-dlbcps])\S*\s+\d+\s+\S+\s+\S+\s+(\d+)\s+\w{3}\s+\d{1,2}\s+(?:\d{1,2}:\d{2}|\d{4})\s(.+)$/.exec(line)
+    if (!match) continue
+    const directory = match[1] === 'd'
+    entries.push({ name: directory ? match[3].replace(/\/$/, '') : match[3], size: Number(match[2]), directory })
+  }
+  return entries
+}
+
+/** Volume number of a split 7z name (`x.7z.001` → 1), 0 for an unsplit `.7z`, null otherwise. */
+export function sevenZipVolume(name: string): number | null {
+  const split = /\.7z\.(\d{1,4})$/i.exec(name)
+  if (split) return Number(split[1])
+  return /\.7z$/i.test(name) ? 0 : null
+}

lib/spotnet/categories.ts

@@ -0,0 +1,333 @@
+/**
+ * Spotnet's category tree, from Spotweb's SpotCategories (English names).
+ * A spot has a head category (0–3) and subcategories per type letter:
+ * `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre,
+ * `d` genre, `z` kind (movie, series, book, …).
+ */
+export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications']
+
+/** What each type letter means per head category. */
+export const SPOT_SUBCAT_TYPES: Record<string, string>[] = [
+  { a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' },
+  { a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' },
+  { a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' },
+  { a: 'Platform', b: 'Genre', z: 'Type' },
+]
+
+const NAMES: Record<string, Record<number, string>>[] = [
+  {
+    a: {
+      0: 'DivX',
+      1: 'WMV',
+      2: 'MPG',
+      3: 'DVD5',
+      4: 'HD other',
+      5: 'ePub',
+      6: 'Blu-ray',
+      7: 'HD-DVD',
+      8: 'WMVHD',
+      9: 'x264',
+      10: 'DVD9',
+      11: 'PDF',
+      12: 'Bitmap',
+      13: 'Vector',
+      14: '3D',
+      15: 'UHD',
+    },
+    b: {
+      0: 'CAM',
+      1: '(S)VCD',
+      2: 'Promo',
+      3: 'Retail',
+      4: 'TV',
+      6: 'Satellite',
+      7: 'R5',
+      8: 'Telecine',
+      9: 'Telesync',
+      10: 'Scan',
+      11: 'WEB-DL',
+      12: 'WEBRip',
+      13: 'HDRip',
+    },
+    c: {
+      0: 'No subtitles',
+      1: 'Dutch subtitles (external)',
+      2: 'Dutch subtitles (builtin)',
+      3: 'English subtitles (external)',
+      4: 'English subtitles (builtin)',
+      6: 'Dutch subtitles (available)',
+      7: 'English subtitles (available)',
+      10: 'English audio/written',
+      11: 'Dutch audio/written',
+      12: 'German audio/written',
+      13: 'French audio/written',
+      14: 'Spanish audio/written',
+      15: 'Asian audio/written',
+    },
+    d: {
+      0: 'Action',
+      1: 'Adventure',
+      2: 'Animation',
+      3: 'Cabaret',
+      4: 'Comedy',
+      5: 'Crime',
+      6: 'Documentary',
+      7: 'Drama',
+      8: 'Family',
+      9: 'Fantasy',
+      10: 'Arthouse',
+      11: 'Television',
+      12: 'Horror',
+      13: 'Music',
+      14: 'Musical',
+      15: 'Mystery',
+      16: 'Romance',
+      17: 'Science Fiction',
+      18: 'Sport',
+      19: 'Short movie',
+      20: 'Thriller',
+      21: 'War',
+      22: 'Western',
+      23: 'Erotica (hetero)',
+      24: 'Erotica (gay male)',
+      25: 'Erotica (gay female)',
+      26: 'Erotica (bi)',
+      28: 'Asian',
+      29: 'Anime',
+      30: 'Cover',
+      31: 'Comicbook',
+      32: 'Cartoons',
+      33: 'Youth',
+      34: 'Business',
+      35: 'Computer',
+      36: 'Hobby',
+      37: 'Cooking',
+      38: 'Handwork',
+      39: 'Craftwork',
+      40: 'Health',
+      41: 'History',
+      42: 'Psychology',
+      43: 'Newspaper',
+      44: 'Magazine',
+      45: 'Science',
+      46: 'Female',
+      47: 'Religion',
+      48: 'Roman',
+      49: 'Biography',
+      50: 'Detective',
+      51: 'Animals',
+      52: 'Humor',
+      53: 'Travel',
+      54: 'True story',
+      55: 'Non-fiction',
+      56: 'Politics',
+      57: 'Poetry',
+      58: 'Fairy tale',
+      59: 'Technical',
+      60: 'Art',
+      72: 'Bi',
+      73: 'Lesbian',
+      74: 'Homo',
+      75: 'Hetero',
+      76: 'Amature',
+      77: 'Group',
+      78: 'POV',
+      79: 'Solo',
+      80: 'Young',
+      81: 'Soft',
+      82: 'Fetish',
+      83: 'Old',
+      84: 'Fat',
+      85: 'SM',
+      86: 'Rough',
+      87: 'Dark',
+      88: 'Hentai',
+      89: 'Outside',
+    },
+    z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' },
+  },
+  {
+    a: {
+      0: 'MP3',
+      1: 'WMA',
+      2: 'WAV',
+      3: 'OGG',
+      4: 'EAC',
+      5: 'DTS',
+      6: 'AAC',
+      7: 'APE',
+      8: 'FLAC',
+    },
+    b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' },
+    c: {
+      0: 'Variable',
+      1: '< 96kbit',
+      2: '96kbit',
+      3: '128kbit',
+      4: '160kbit',
+      5: '192kbit',
+      6: '256kbit',
+      7: '320kbit',
+      8: 'Lossless',
+      9: 'Other',
+    },
+    d: {
+      0: 'Blues',
+      1: 'Compilation',
+      2: 'Cabaret',
+      3: 'Dance',
+      4: 'Diverse',
+      5: 'Hardcore',
+      6: 'World',
+      7: 'Jazz',
+      8: 'Youth',
+      9: 'Classical',
+      10: 'Kleinkunst',
+      11: 'Dutch',
+      12: 'New Age',
+      13: 'Pop',
+      14: 'RnB',
+      15: 'Hiphop',
+      16: 'Reggae',
+      17: 'Religious',
+      18: 'Rock',
+      19: 'Soundtracks',
+      20: 'Other',
+      21: 'Hardstyle',
+      22: 'Asian',
+      23: 'Disco',
+      24: 'Classics',
+      25: 'Metal',
+      26: 'Country',
+      27: 'Dubstep',
+      28: 'Nederhop',
+      29: 'DnB',
+      30: 'Electro',
+      31: 'Folk',
+      32: 'Soul',
+      33: 'Trance',
+      34: 'Balkan',
+      35: 'Techno',
+      36: 'Ambient',
+      37: 'Latin',
+      38: 'Live',
+    },
+    z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' },
+  },
+  {
+    a: {
+      0: 'Windows',
+      1: 'Macintosh',
+      2: 'Linux',
+      3: 'Playstation',
+      4: 'Playstation 2',
+      5: 'PSP',
+      6: 'Xbox',
+      7: 'Xbox 360',
+      8: 'Gameboy Advance',
+      9: 'Gamecube',
+      10: 'Nintendo DS',
+      11: 'Nintento Wii',
+      12: 'Playstation 3',
+      13: 'Windows Phone',
+      14: 'iOS',
+      15: 'Android',
+      16: 'Nintendo 3DS',
+      17: 'Playstation 4',
+      18: 'XBox 1',
+    },
+    b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' },
+    c: {
+      0: 'Action',
+      1: 'Adventure',
+      2: 'Strategy',
+      3: 'Roleplaying',
+      4: 'Simulation',
+      5: 'Race',
+      6: 'Flying',
+      7: 'Shooter',
+      8: 'Platform',
+      9: 'Sport',
+      10: 'Child/youth',
+      11: 'Puzzle',
+      12: 'Other',
+      13: 'Boardgame',
+      14: 'Cards',
+      15: 'Education',
+      16: 'Music',
+      17: 'Family',
+    },
+  },
+  {
+    a: {
+      0: 'Windows',
+      1: 'Macintosh',
+      2: 'Linux',
+      3: 'OS/2',
+      4: 'Windows Phone',
+      5: 'Navigation systems',
+      6: 'iOS',
+      7: 'Android',
+    },
+    b: {
+      0: 'Audio',
+      1: 'Video',
+      2: 'Graphics',
+      3: 'CD/DVD Tools',
+      4: 'Media players',
+      5: 'Rippers &amp; Encoders',
+      6: 'Plugins',
+      7: 'Database tools',
+      8: 'Email software',
+      9: 'Photo',
+      10: 'Screensavers',
+      11: 'Skin software',
+      12: 'Drivers',
+      13: 'Browsers',
+      14: 'Download managers',
+      15: 'Download',
+      16: 'Usenet software',
+      17: 'RSS Readers',
+      18: 'FTP software',
+      19: 'Firewalls',
+      20: 'Antivirus software',
+      21: 'Antispyware software',
+      22: 'Optimization software',
+      23: 'Security software',
+      24: 'System software',
+      25: 'Other',
+      26: 'Educational',
+      27: 'Office',
+      28: 'Internet',
+      29: 'Communication',
+      30: 'Development',
+      31: 'Spotnet',
+    },
+  },
+]
+
+export interface SpotCategoryLabels {
+  head: string | null
+  /** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */
+  subcats: { type: string; values: string[] }[]
+}
+
+/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */
+export function spotCategoryLabels(
+  category: number | null,
+  subcats: Record<string, number[]> | null
+): SpotCategoryLabels {
+  if (category === null || !(category in NAMES)) return { head: null, subcats: [] }
+  const labels: SpotCategoryLabels['subcats'] = []
+  for (const letter of ['z', 'a', 'b', 'c', 'd']) {
+    const values = [
+      ...new Set(
+        (subcats?.[letter] ?? [])
+          .map((n) => NAMES[category][letter]?.[n])
+          .filter((v): v is string => Boolean(v))
+      ),
+    ]
+    if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values })
+  }
+  return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels }
+}

lib/spotnet/index.ts

@@ -4,4 +4,6 @@
  * See CONCEPT.md (D29).
  */
 export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
-export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, type SpotXml } from './xml.js'
+export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, decodeSpotText, type SpotXml, type SpotImage } from './xml.js'
+export { spotCategoryLabels, SPOT_HEAD_CATEGORIES, type SpotCategoryLabels } from './categories.js'
+export { passwordFromSpot } from './password.js'

lib/spotnet/password.ts

@@ -0,0 +1,50 @@
+/**
+ * Archive passwords that spotters put in the title or description instead of
+ * the NZB's `<meta type="password">`:
+ *   `Title{{secret}}` (the newznab/SABnzbd convention),
+ *   `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`,
+ *   `WW FC0401` (abbreviation without separator: only with a digit in it).
+ * The value must be the rest of its line and a single token, so sentences
+ * such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or
+ * "met een password beschermd" don't count.
+ */
+const KEYWORD = String.raw`wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw`
+const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])(?:${KEYWORD})\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
+const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i
+const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i
+const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i
+
+/** Lines of a spot description: UBB/HTML line breaks split, other tags dropped. */
+function lines(description: string): string[] {
+  return description
+    .replace(/\[br\]|<br\s*\/?>/gi, '\n')
+    .replace(/\[\/?[a-z]+(?:=[^\]]*)?\]|<\/?[a-z][^>]*>/gi, '')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+}
+
+function clean(value: string): string | null {
+  const token = value.trim().replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
+  if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null
+  return token
+}
+
+export function passwordFromSpot(title: string, description?: string | null): string | null {
+  const braces = /\{\{([^{}]{1,64})\}\}/.exec(title)
+  if (braces && clean(braces[1])) return clean(braces[1])
+  for (const line of lines(description ?? '')) {
+    if (!line || NO_PASSWORD.test(line)) continue
+    const separated = WITH_SEPARATOR.exec(line)
+    if (separated) {
+      const value = clean(separated[1])
+      if (value) return value
+      continue
+    }
+    const abbreviated = ABBREVIATION.exec(line)
+    if (abbreviated && /\d/.test(abbreviated[1])) {
+      const value = clean(abbreviated[1])
+      if (value) return value
+    }
+  }
+  return null
+}

lib/spotnet/xml.ts

@@ -6,18 +6,25 @@ export interface SpotXml {
   subcats: string[]
   size: number
   website: string
+  /** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
+  description: string
+  tag: string
+  /** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
+  image: SpotImage | null
   /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
   nzbSegments: string[]
   created: number | null
 }
 
+export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }
+
 const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
 
+/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
 function text(value: string | undefined): string {
   if (value === undefined) return ''
   const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
-  if (cdata) return cdata[1]
-  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
+  return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
     if (entity[0] === '#') {
       const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
       return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
@@ -37,6 +44,9 @@ export function parseSpotXml(xml: string): SpotXml | null {
   const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
   const nzb = element(posting, 'NZB') ?? ''
   return {
+    description: text(element(posting, 'Description')).trim(),
+    tag: text(element(posting, 'Tag')).trim(),
+    image: parseImage(posting),
     title: text(element(posting, 'Title')).trim(),
     // Newer keys number categories from 1.
     category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
@@ -48,15 +58,52 @@ export function parseSpotXml(xml: string): SpotXml | null {
   }
 }
 
-/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */
+function parseImage(posting: string): SpotImage | null {
+  const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
+  if (!match) return null
+  const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
+  if (segments.length > 0) {
+    const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
+    return { segments, width: size('Width'), height: size('Height') }
+  }
+  const url = text(match[2]).trim()
+  return /^https?:\/\//i.test(url) ? { url } : null
+}
+
+/**
+ * The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
+ * the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
+ */
+export function decodeSpotText(latin1: string): string {
+  const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
+  return utf8.includes('\uFFFD') ? latin1 : utf8
+}
+
+/**
+ * Header lines of a raw HEAD response; X-XML (split over many lines) joined.
+ * Folded headers are unfolded: old (key 1) spots put the From address on a
+ * continuation line.
+ */
 export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
   const headers: Record<string, string> = {}
   let xml = ''
+  let last: string | null = null
+  let lastIsFirst = false
   for (const line of raw.split(/\r?\n/)) {
+    if (/^[ \t]/.test(line) && last) {
+      if (last === 'x-xml') xml += line
+      else if (lastIsFirst) headers[last] += line
+      continue
+    }
     const colon = line.indexOf(':')
-    if (colon <= 0) continue
+    if (colon <= 0) {
+      last = null
+      continue
+    }
     const name = line.slice(0, colon).toLowerCase()
     const value = line.slice(colon + 1).replace(/^ /, '')
+    last = name
+    lastIsFirst = !(name in headers)
     if (name === 'x-xml') xml += value
     else headers[name] ??= value
   }

lib/subject/file_key.ts

@@ -28,3 +28,20 @@ export function weakFileKey(name: string, size: number, totalParts: number): str
   const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
   return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
 }
+
+/**
+ * Identity of a file by its counters, for uploads that give every article
+ * its own random subject (CONCEPT.md D36): poster, file counter `[i/M]`,
+ * part count and posting day stay the same across the articles of one file.
+ * The day keeps different uploads with the same shape apart; collisions that
+ * remain are caught by the conflict check (these files are weak identities).
+ * Null when the subject has no file counter.
+ */
+export function counterFileKey(segment: { poster: string; subject: string; totalParts: number; postedAt?: Date }): string | null {
+  const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject)
+  if (!counter || segment.totalParts < 2) return null
+  const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : ''
+  return createHash('sha1')
+    .update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
+    .digest('hex')
+}

lib/subject/index.ts

@@ -2,4 +2,4 @@
  * Conventions of binary posts (subject lines, file identity), independent of AdonisJS.
  */
 export { parseSubject, type ParsedSubject } from './parse.js'
-export { fileKey, weakFileKey, looksLikeFilename } from './file_key.js'
+export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'

public/app.css

@@ -263,6 +263,41 @@ kbd {
   font-size: 13px;
 }
 
+/* While a new search runs, the previous results stay in place but greyed out. */
+.release-list.stale {
+  opacity: 0.4;
+  pointer-events: none;
+  transition: opacity 0.12s ease-out;
+}
+
+.results[aria-busy='true'] .summary::after {
+  content: '';
+  display: block;
+  height: 2px;
+  margin-top: 6px;
+  border-radius: 1px;
+  background: linear-gradient(90deg, transparent, var(--accent), transparent);
+  background-size: 40% 100%;
+  background-repeat: no-repeat;
+  animation: searching 1s linear infinite;
+}
+
+@keyframes searching {
+  from {
+    background-position: -40% 0;
+  }
+  to {
+    background-position: 140% 0;
+  }
+}
+
+@media (prefers-reduced-motion: reduce) {
+  .results[aria-busy='true'] .summary::after {
+    animation: none;
+    background: var(--accent-soft);
+  }
+}
+
 .release-list {
   list-style: none;
   margin: 0;
@@ -563,6 +598,70 @@ kbd {
   color: var(--bad);
 }
 
+.notice-good {
+  background: var(--good-soft);
+  color: var(--good);
+}
+
+.notice .password {
+  padding: 1px 6px;
+  border-radius: 4px;
+  background: var(--surface);
+  color: var(--text);
+  font-family: var(--mono);
+  user-select: all;
+  overflow-wrap: anywhere;
+}
+
+.spot {
+  display: flex;
+  gap: 16px;
+  align-items: flex-start;
+}
+
+.spot-image {
+  flex: none;
+  width: 140px;
+  height: auto;
+  border-radius: 6px;
+  background: var(--surface-2);
+}
+
+.spot-body {
+  flex: 1;
+  min-width: 0;
+}
+
+.spot-body .facts {
+  margin-bottom: 14px;
+}
+
+.spot-description {
+  font-size: 13.5px;
+  line-height: 1.55;
+  overflow-wrap: anywhere;
+}
+
+.spot-description a,
+.spot-links a {
+  color: var(--accent);
+}
+
+.spot-links {
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+  margin-top: 12px;
+  font-size: 13px;
+  overflow-wrap: anywhere;
+}
+
+@media (max-width: 560px) {
+  .spot {
+    flex-direction: column;
+  }
+}
+
 .notice-warn {
   background: var(--warn-soft);
   color: var(--warn);

public/app.js

@@ -27,6 +27,7 @@ const state = {
   release: null,
 }
 let requestId = 0
+let searchAbort = null
 
 // --- Formatting ------------------------------------------------------------
 
@@ -118,8 +119,8 @@ function syncControls() {
 
 // --- API -------------------------------------------------------------------
 
-async function api(path) {
-  const response = await fetch(path, { headers: { accept: 'application/json' } })
+async function api(path, { signal } = {}) {
+  const response = await fetch(path, { headers: { accept: 'application/json' }, signal })
   if (!response.ok) throw new Error(`${response.status} ${response.statusText}`)
   return response.json()
 }
@@ -139,11 +140,13 @@ async function loadStats() {
   try {
     const stats = await api('/api/v1/stats')
     const readable = stats.releases ? Math.round((100 * stats.readableReleases) / stats.releases) : 0
+    // Estimated from Postgres statistics and a sample (CONCEPT.md D33).
+    const about = stats.approximate ? '≈ ' : ''
     const items = [
-      ['Releases', number.format(stats.releases)],
-      ['Readable', `${readable}%`],
-      ['Files', number.format(stats.files)],
-      ['Size', formatSize(stats.size)],
+      ['Releases', about + number.format(stats.releases)],
+      ['Readable', `${about}${readable}%`],
+      ['Files', about + number.format(stats.files)],
+      ['Size', about + formatSize(stats.size)],
       ['Indexed', formatAge(stats.lastIndexedAt) || '–'],
     ]
     $('#stats').replaceChildren(...items.map(([dt, dd]) => el('div', {}, el('dt', { text: dt }), el('dd', { text: dd }))))
@@ -174,9 +177,14 @@ function releaseRow(release) {
   if (release.par2Only) badges.append(badge('par2 only', 'warn', 'Only the par2 files arrived; the data they describe is missing'))
   if (release.container === 'nzbe') badges.append(badge('Encrypted NZB', 'bad', 'An encrypted NZB container: unusable without its key'))
   else if (release.container === 'rclone') badges.append(badge('Encrypted (rclone)', 'bad', 'rclone crypt: names and contents encrypted, unusable without the key'))
-  else if (release.encrypted) badges.append(badge('Password', 'bad', 'Password-protected archive'))
+  else if (release.container === 'yencbin') badges.append(badge('Encrypted (.bin)', 'bad', 'An encrypted .bin container (yEncBin): unusable without its key'))
+  else if (release.passwordState === 'missing') badges.append(badge('Password needed', 'bad', 'Password-protected archive; the password is not known'))
+  else if (release.passwordState === 'known') {
+    badges.append(badge('Password known', 'good', 'The password is in the NZB; SABnzbd and NZBGet use it to unpack'))
+  }
   const files = `${release.fileCount} ${release.fileCount === 1 ? 'file' : 'files'}`
-  if (release.complete) badges.append(badge(files, 'good', 'Complete'))
+  if (release.listingOnly) badges.append(badge('Spot', 'accent', 'A Spotnet spot: its NZB is fetched from Usenet when you download it'))
+  else if (release.complete) badges.append(badge(files, 'good', 'Complete'))
   else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced'))
   else if (release.listedFiles) {
     badges.append(badge(`${release.listedPresent}/${release.listedFiles} listed`, 'warn', 'Files the par2 set lists: present / total'))
@@ -195,24 +203,37 @@ function releaseRow(release) {
 async function loadResults({ append = false } = {}) {
   const id = ++requestId
   if (!append) state.page = 1
+  // A newer search makes the running one pointless: cancel it.
+  searchAbort?.abort()
+  const abort = (searchAbort = new AbortController())
   $('#more').hidden = true
-  if (!append) $('#summary').textContent = 'Searching…'
+  if (!append) {
+    $('#summary').textContent = 'Searching…'
+    $('#empty').hidden = true
+    // The old rows no longer match the search; grey them out until the new ones arrive.
+    $('#list').classList.add('stale')
+    $('.results').setAttribute('aria-busy', 'true')
+  }
 
   try {
-    const result = await api(`/api/v1/releases?${searchParams(state.page)}`)
+    const result = await api(`/api/v1/releases?${searchParams(state.page)}`, { signal: abort.signal })
     if (id !== requestId) return // a newer search superseded this one
+    $('#list').classList.remove('stale')
+    $('.results').removeAttribute('aria-busy')
     const rows = result.releases.map(releaseRow)
     if (append) $('#list').append(...rows)
     else $('#list').replaceChildren(...rows)
 
     const shown = $('#list').children.length
     $('#summary').textContent =
-      result.total === 0 ? '' : `${number.format(result.total)} ${result.total === 1 ? 'release' : 'releases'}` +
+      result.total === 0 ? '' : `${number.format(result.total)}${result.totalCapped ? '+' : ''} ${result.total === 1 ? 'release' : 'releases'}` +
         (shown < result.total ? ` · showing ${number.format(shown)}` : '')
     $('#empty').hidden = result.total !== 0
-    $('#more').hidden = shown >= result.total
+    $('#more').hidden = shown >= result.total && !result.totalCapped
   } catch (error) {
     if (id !== requestId) return
+    $('#list').classList.remove('stale')
+    $('.results').removeAttribute('aria-busy')
     $('#summary').textContent = `Search failed: ${error.message}`
   }
 }
@@ -237,6 +258,80 @@ function section(title, note, content) {
   return [el('h3', { class: 'section-title' }, title, note ? el('small', { text: note }) : null), content]
 }
 
+/**
+ * Spotnet descriptions use UBB markup: [br], [b], [i], [u], [url=…]…[/url].
+ * Built as DOM nodes (never HTML); links only when they are http(s).
+ */
+function ubb(text) {
+  const root = el('div', { class: 'spot-description' })
+  const stack = [root]
+  const top = () => stack[stack.length - 1]
+  // Old (key 1) spots use HTML instead: line breaks and simple tags become UBB, the rest is dropped.
+  text = text
+    .replace(/<br\s*\/?>/gi, '[br]')
+    .replace(/<(\/?)(b|i|u)>/gi, '[$1$2]')
+    .replace(/<\/?[a-z][^>]*>/gi, '')
+  for (const token of text.split(/(\[\/?[a-z]+(?:=[^\]]*)?\])/i)) {
+    const tag = /^\[(\/?)([a-z]+)(?:=([^\]]*))?\]$/i.exec(token)
+    if (!tag) {
+      if (token) top().append(token)
+      continue
+    }
+    const [, closing, rawName, arg] = tag
+    const name = rawName.toLowerCase()
+    if (name === 'br') {
+      top().append(el('br'))
+    } else if (!closing && ['b', 'i', 'u', 'url'].includes(name)) {
+      const url = name === 'url' ? (arg ?? '').trim() : ''
+      const node = name === 'url'
+        ? /^https?:\/\//i.test(url) ? el('a', { href: url, target: '_blank', rel: 'noopener noreferrer nofollow' }) : el('span')
+        : el({ b: 'strong', i: 'em', u: 'u' }[name])
+      node.dataset.ubb = name
+      top().append(node)
+      stack.push(node)
+    } else if (closing && stack.length > 1 && top().dataset.ubb === name) {
+      stack.pop()
+    }
+    // Other tags ([img], [color=…], …) are dropped.
+  }
+  return root
+}
+
+function spotSection(release) {
+  const spot = release.spot
+  if (!spot) return []
+  const image = spot.image?.url
+    ? el('img', {
+        class: 'spot-image',
+        src: spot.image.url,
+        alt: '',
+        loading: 'lazy',
+        width: spot.image.width ?? undefined,
+        height: spot.image.height ?? undefined,
+        onerror: (event) => event.target.remove(),
+      })
+    : null
+  const categories = spot.categories?.head
+    ? el('dl', { class: 'facts' },
+        fact('Category', spot.categories.head),
+        ...spot.categories.subcats.map((sub) => fact(sub.type, sub.values.join(', '))))
+    : null
+  const links = el('div', { class: 'spot-links' },
+    spot.website && /^https?:\/\//i.test(spot.website)
+      ? el('a', { href: spot.website, target: '_blank', rel: 'noopener noreferrer nofollow', text: spot.website })
+      : null,
+    spot.image?.external
+      ? el('a', { href: spot.image.external, target: '_blank', rel: 'noopener noreferrer nofollow', text: 'Image (external site)' })
+      : null)
+  const note = [spot.poster && `by ${spot.poster}`, spot.tag && spot.tag !== spot.poster && `tag ${spot.tag}`].filter(Boolean).join(' · ')
+  return section('Spot', note, el('div', { class: 'spot' },
+    image,
+    el('div', { class: 'spot-body' },
+      categories,
+      spot.description ? ubb(spot.description) : el('p', { class: 'empty', text: 'No description.' }),
+      links.childElementCount ? links : null)))
+}
+
 function renderDetail(release) {
   $('#detail-name').textContent = release.name
   $('#detail-source').textContent = `Recognised by ${SOURCE_LABELS[release.source] ?? release.source ?? 'filename'}`
@@ -289,10 +384,22 @@ function renderDetail(release) {
   const notices = []
   if (release.container === 'nzbe') {
     notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted NZB container ("nzbe"): an encrypted download index for others, unreadable without its key.' }))
+  } else if (release.container === 'yencbin') {
+    notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted .bin container (from the yEncBin uploader): the data is encrypted with a key that is not part of the post.' }))
   } else if (release.container === 'rclone') {
     notices.push(el('div', { class: 'notice notice-bad', text: 'rclone crypt: file names and contents are encrypted with a password and salt that are not part of the post.' }))
-  } else if (release.encrypted) {
-    notices.push(el('div', { class: 'notice notice-bad', text: 'Password-protected: the archive needs a password that is not part of the post.' }))
+  } else if (release.passwordState === 'missing') {
+    notices.push(el('div', { class: 'notice notice-bad', text: 'Password needed: the archive is password-protected, and neither the post nor its spot says the password.' }))
+  }
+  if (release.password) {
+    notices.push(el('div', { class: 'notice notice-good' },
+      el('div', {}, 'Password ', el('code', { class: 'password', text: release.password }),
+        release.passwordSource === 'spot' ? ' (from the spot\'s description)' : '',
+        ` is included in the NZB; SABnzbd and NZBGet use it to unpack.${release.encrypted == null ? ' Whether the archive needs it is not known yet.' : ''}`)))
+  }
+  if (release.listingOnly) {
+    notices.push(el('div', { class: 'notice' },
+      `A Spotnet spot${release.spot?.poster ? ` by ${release.spot.poster}` : ''}. Its files are read from the spot's NZB on Usenet when you download it; that takes a few seconds the first time.`))
   }
   if (release.pending.binaries > 0) {
     const pct = release.pending.partsTotal ? Math.round((100 * release.pending.partsHave) / release.pending.partsTotal) : 0
@@ -334,6 +441,7 @@ function renderDetail(release) {
     actions,
     ...notices,
     facts,
+    ...spotSection(release),
     ...section('Files', `${release.files.length} posted`, files),
     ...(par2 ? section('par2 file list', `${listedPresent} of ${release.par2Files.length} present`, par2) : []),
     ...(contents ? section('Archive contents', `${release.contents.length} entries`, contents) : [])

public/index.html

@@ -39,7 +39,7 @@
         <div class="filters" role="group" aria-label="Filters">
           <button class="chip" type="button" data-filter="readable" aria-pressed="false">Readable names</button>
           <button class="chip" type="button" data-filter="complete" aria-pressed="false">Complete</button>
-          <button class="chip" type="button" data-filter="hide_encrypted" aria-pressed="false">Hide passworded</button>
+          <button class="chip" type="button" data-filter="hide_encrypted" aria-pressed="false" title="Hide releases that need a password we don't know">Hide password needed</button>
           <button class="chip" type="button" data-filter="par2_only" aria-pressed="false" title="Also show releases that are only par2 files whose data never arrived">Show par2-only</button>
           <label class="select select-small">
             <span class="visually-hidden">Recognised by</span>

start/env.ts

@@ -47,8 +47,18 @@ export default await Env.create(new URL('../', import.meta.url), {
   NNTP_PASSWORD: Env.schema.secret(),
   NNTP_SECURE: Env.schema.boolean(),
   NNTP_POOL_SIZE: Env.schema.number.optional(),
+  NNTP_COMMAND_TIMEOUT_MS: Env.schema.number.optional(),
 
   // Indexer (config/indexer.ts)
+  BACKFILL_MAX_QUEUED: Env.schema.number.optional(),
+  STAGING_GROUPS: Env.schema.string.optional(),
+  COUNTER_GROUPING_GROUPS: Env.schema.string.optional(),
+  FILE_CONCURRENCY: Env.schema.number.optional(),
+  GC_REINDEX_INTERVAL_MS: Env.schema.number.optional(),
+  GC_UNFINGERPRINTED_SECONDS: Env.schema.number.optional(),
+  STAGING_TTL_SECONDS: Env.schema.number.optional(),
+  BACKFILL_MAX_NVME_GB: Env.schema.number.optional(),
+  BACKFILL_MAX_HDD_GB: Env.schema.number.optional(),
   INDEX_INTERVAL_MS: Env.schema.number.optional(),
   INDEX_BATCH_SIZE: Env.schema.number.optional(),
   INDEX_MAX_BATCHES_PER_TICK: Env.schema.number.optional(),
@@ -60,6 +70,7 @@ export default await Env.create(new URL('../', import.meta.url), {
   DEFERRED_EXPAND_MAX_PARTS: Env.schema.number.optional(),
   DEFERRED_CONCURRENCY: Env.schema.number.optional(),
   SPOT_CONCURRENCY: Env.schema.number.optional(),
+  SPOT_IMPORT_DAYS: Env.schema.number.optional(),
 
   // Newznab API (/api): required as ?apikey= when set
   NEWZNAB_API_KEY: Env.schema.secret.optional(),

start/routes.ts

@@ -23,6 +23,7 @@ router
     router.get('releases', [ReleasesController, 'index'])
     router.get('releases/:id', [ReleasesController, 'show']).where('id', router.matchers.number())
     router.get('releases/:id/nzb', [ReleasesController, 'nzb']).where('id', router.matchers.number())
+    router.get('releases/:id/image', [ReleasesController, 'image']).where('id', router.matchers.number())
     router.get('stats', [ReleasesController, 'stats'])
   })
   .prefix('/api/v1')

tests/bootstrap.ts

@@ -42,7 +42,13 @@ export const plugins: Config['plugins'] = [
  */
 export const runnerHooks: Required<Pick<Config, 'setup' | 'teardown'>> = {
   setup: [],
-  teardown: [],
+  teardown: [
+    // Tests that complete a file queue jobs; their Redis connections would keep the process alive.
+    async () => {
+      const { default: QueueService } = await import('#services/QueueService')
+      await QueueService.closeAll()
+    },
+  ],
 }
 
 /**

tests/functional/backfill.spec.ts

@@ -0,0 +1,38 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import indexerConfig from '#config/indexer'
+import BackfillService from '#services/BackfillService'
+import FetchRangeService from '#services/FetchRangeService'
+
+test.group('BackfillService', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('queues header-only ranges downwards, capped, without holding back the live watermark', async ({ assert }) => {
+    const now = new Date()
+    const batch = Number(indexerConfig.batchSize)
+    const [g] = await db
+      .table('groups')
+      .insert({
+        name: 'alt.binaries.backfilltest', active: true, fetch_bodies: true, last_queued_id: 5_000_000, last_indexed_id: 5_000_000,
+        backfill_until_id: 5_000_000 - 100 * batch, backfill_next_id: 4_900_000, created_at: now, updated_at: now,
+      })
+      .returning('*')
+
+    const queued = await BackfillService.tick()
+    assert.equal(queued, indexerConfig.backfill.maxQueued)
+    const ranges = await db.from('fetch_ranges').where('group_id', g.id).orderBy('end_id', 'desc')
+    assert.isTrue(ranges.every((r) => r.backfill && r.status === 'queued'))
+    assert.equal(Number(ranges[0].end_id), 4_900_000)
+    assert.equal(Number(ranges[0].start_id), 4_900_000 - batch + 1)
+    const after = await db.from('groups').where('id', g.id).first()
+    assert.equal(Number(after.backfill_next_id), 4_900_000 - queued * batch)
+
+    // The cap holds while those wait.
+    assert.equal(await BackfillService.tick(), 0)
+
+    // Backfill ranges don't hold back the live watermark.
+    await db.from('groups').where('id', g.id).update({ last_queued_id: 5_100_000 })
+    assert.equal(await FetchRangeService.advanceWatermark({ id: g.id, name: g.name }), 5_100_000n)
+  })
+})

tests/functional/counter_grouping.spec.ts

@@ -0,0 +1,70 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import StagingService from '#services/StagingService'
+import IngestService, { segmentFromHeader } from '#services/IngestService'
+import { counterFileKey } from '#lib/subject/index'
+
+let n = 0
+/** An article of a random-subject upload: a new random name every time. */
+const randomArticle = (k: number, total = 4, { counter = '[21/62]', date = 'Thu, 01 Oct 2026 12:00:00 GMT', id }: { counter?: string; date?: string; id?: string } = {}) =>
+  segmentFromHeader(
+    { 'subject': `${counter} "Rnd${++n}xQ7kLm2Pz9Wv4Ty8Ab3Cd" yEnc (${k}/${total})`, 'from': 'bob@home.com', 'message-id': `<${id ?? `art${n}`}@nyuu>`, 'date': date, ':bytes': '700000' },
+    'alt.binaries.boneless',
+    k
+  )!
+
+async function ingestBatch(segments: ReturnType<typeof randomArticle>[], counters = true) {
+  const r = await StagingService.filter(segments, { counters })
+  await IngestService.ingest(r.ingest)
+  return r
+}
+
+test.group('Counter grouping', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('articles with random subjects become one file by poster, counter and part count', async ({ assert }) => {
+    const key = counterFileKey(randomArticle(1))!
+    await ingestBatch([randomArticle(1)])
+    assert.isNull(await db.from('binaries').where('file_key', key).first())
+    await ingestBatch([randomArticle(2)])
+    const binary = await db.from('binaries').where('file_key', key).first()
+    assert.isNotNull(binary)
+    // Later articles join the file directly, until it completes.
+    await ingestBatch([randomArticle(3)])
+    await ingestBatch([randomArticle(4)])
+    const parts = await db.from('parts').where('binary_id', binary.id).orderBy('number').select('number')
+    assert.deepEqual(parts.map((p) => p.number), [1, 2, 3, 4])
+    assert.isNotNull((await db.from('binaries').where('id', binary.id).first()).completed_at)
+  })
+
+  test('off by default; another day is another file; a repeated part number is a conflict', async ({ assert }) => {
+    await ingestBatch([randomArticle(1, 9, { counter: '[05/10]' })], false)
+    await ingestBatch([randomArticle(2, 9, { counter: '[05/10]' })], false)
+    assert.isNull(await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[05/10]' }))!).first())
+
+    const other = randomArticle(1, 9, { counter: '[06/10]', date: 'Fri, 02 Oct 2026 12:00:00 GMT' })
+    assert.notEqual(counterFileKey(other), counterFileKey(randomArticle(1, 9, { counter: '[06/10]' })))
+
+    // Two uploads of the same shape on one day: part 1 twice, from different articles.
+    await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
+    await ingestBatch([randomArticle(2, 9, { counter: '[07/10]' })])
+    await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
+    const binary = await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[07/10]' }))!).first()
+    assert.isNotNull(binary.conflicted_at)
+  })
+
+  test('a normal file with a counter stays a normal file', async ({ assert }) => {
+    const normal = (k: number) =>
+      segmentFromHeader(
+        { 'subject': `[07/45] - "Some.Release.part06.rar" yEnc (${k}/3)`, 'from': 'poster@x', 'message-id': `<norm${k}@x>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
+        'alt.binaries.test',
+        k
+      )!
+    await ingestBatch([normal(1)])
+    await ingestBatch([normal(2), normal(3)])
+    const binary = await db.from('binaries').where('name', 'Some.Release.part06.rar').first()
+    assert.equal(binary.file_key, normal(1).fileKey)
+    assert.isNotNull(binary.completed_at)
+  })
+})

tests/functional/names.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
 import testUtils from '@adonisjs/core/services/test_utils'
 import NameService from '#services/NameService'
 
@@ -36,10 +37,9 @@ test.group('NameService', (group) => {
       .table('releases')
       .insert({ release_key: 'w'.repeat(40), name: 'mOzyyxCyYUjAbCdEf12', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
-    const [file] = await db
-      .table('files')
-      .insert({ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', message_ids: '{}', release_id: weak.id, created_at: now, updated_at: now })
-      .returning('id')
+    const [file] = await FileStore.insert([
+      { filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], release_id: weak.id, created_at: now, updated_at: now },
+    ])
     const episode = await binary('ep18', { size: 939_684_767, md5_16k: 'd'.repeat(32), release_id: weak.id, file_id: file.id })
 
     await NameService.resolve([episode])

tests/functional/newznab.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
 import env from '#start/env'
 import testUtils from '@adonisjs/core/services/test_utils'
 
@@ -11,11 +12,11 @@ async function seed(name: string, opts: { fileTotal?: number; files?: number } =
     .insert({ release_key: `nz${++counter}`.padEnd(40, 'n'), name, source: 'stem', poster: 'poster@example.invalid', groups: '["alt.binaries.test"]', file_total: opts.fileTotal ?? null, posted_at: now, first_seen_at: now, last_seen_at: now })
     .returning('id')
   for (let i = 0; i < (opts.files ?? 1); i++) {
-    await db.table('files').insert({
+    await FileStore.insert([{
       filename: `${name}.part${i + 1}.rar`, poster: 'poster@example.invalid', date: now.getTime(), parts: 1, size: 1_000_000,
-      release_id: release.id, groups: '["alt.binaries.test"]', message_ids: JSON.stringify({ '1': { id: `${name}-${i}@test`, size: 1_000_000 } }),
+      release_id: release.id, groups: '["alt.binaries.test"]', segments: [{ number: 1, id: `${name}-${i}@test`, size: 1_000_000 }],
       created_at: now, updated_at: now,
-    })
+    }])
   }
   return Number(release.id)
 }

tests/functional/nzb_import.spec.ts

@@ -1,5 +1,7 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
+import type { Segment } from '#lib/nzb/index'
 import testUtils from '@adonisjs/core/services/test_utils'
 import NzbImportService, { nzbFileName } from '#services/NzbImportService'
 import SearchService from '#services/SearchService'
@@ -17,13 +19,10 @@ const XML = `<?xml version="1.0" encoding="utf-8"?>
   </file>
 </nzb>`
 
-async function file(fields: Record<string, unknown>) {
+async function file(fields: Record<string, unknown> & { segments: Segment[] }) {
   const now = new Date()
-  const [row] = await db
-    .table('files')
-    .insert({ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields })
-    .returning('*')
-  return row
+  const [{ id }] = await FileStore.insert([{ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }])
+  return db.from('files').where('id', id).first()
 }
 
 test.group('NzbImportService', (group) => {
@@ -32,6 +31,14 @@ test.group('NzbImportService', (group) => {
   test('names entries despite odd quoting', ({ assert }) => {
     assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar')
     assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2')
+    // Spot NZBs without quotes.
+    assert.equal(nzbFileName('Adventure.part0055.rar - [056/175] (1/1)'), 'Adventure.part0055.rar')
+    assert.equal(nzbFileName('LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2 (1/0) (1/0)'), 'LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2')
+    assert.equal(nzbFileName('Reacher.S04.vol001+02.par2 yEnc'), 'Reacher.S04.vol001+02.par2')
+    assert.equal(nzbFileName('06 - A New Land, a New Music.flac  Mehmet Ali Sanlikol - The Rise Up (2020) (01/35)'), '06 - A New Land, a New Music.flac')
+    assert.equal(nzbFileName('[01/10] - Some.Release.2026.part01.rar yEnc (1/50) 35000000'), 'Some.Release.2026.part01.rar')
+    assert.equal(nzbFileName('Mozart Zauberfloete 2026 mp4 720p.zip.001'), 'Mozart Zauberfloete 2026 mp4 720p.zip.001')
+    assert.equal(nzbFileName('LibreOffice_7.6.4_Win_x86-64.msi.vol0000'), 'LibreOffice_7.6.4_Win_x86-64.msi.vol0000')
   })
 
   test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => {
@@ -40,8 +47,8 @@ test.group('NzbImportService', (group) => {
       .table('releases')
       .insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
-    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
-    const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', message_ids: JSON.stringify({ '1': { id: 'nzb-1@x', size: 2000 } }) })
+    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', segments: [{ number: 1, id: 'nzb-1@x', size: 2000 }] })
 
     const result = (await NzbImportService.import(posted, XML))!
     assert.equal(result.releaseId, Number(release.id))
@@ -70,7 +77,7 @@ test.group('NzbImportService', (group) => {
         <groups><group>alt.binaries.x</group></groups>
         <segments><segment bytes="500" number="1">${id}@x</segment></segments>
       </file></nzb>`
-    const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', message_ids: JSON.stringify({ '1': { id: 'rar-1@x', size: 2000 } }) })
+    const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', segments: [{ number: 1, id: 'rar-1@x', size: 2000 }] })
 
     const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))!
     const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))!
@@ -87,11 +94,44 @@ test.group('NzbImportService', (group) => {
       (await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id
     const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT')
     const lonely = await insert('lonely', '.nzb')
-    await file({ filename: 'x.part1.rar', release_id: target, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
-    const posted = await file({ filename: '.nzb', release_id: lonely, message_ids: JSON.stringify({ '1': { id: 'nzb-2@x', size: 2000 } }) })
+    await file({ filename: 'x.part1.rar', release_id: target, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    const posted = await file({ filename: '.nzb', release_id: lonely, segments: [{ number: 1, id: 'nzb-2@x', size: 2000 }] })
 
     await NzbImportService.import(posted, XML)
     assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target))
     assert.notExists(await db.from('releases').where('id', lonely).first())
   })
+
+  test('a trusted NZB (a spot) takes its files into its own release, named after it', async ({ assert }) => {
+    const now = new Date()
+    const insertRelease = async (key: string, name: string, source: string) =>
+      (
+        await db
+          .table('releases')
+          .insert({ release_key: key.repeat(40).slice(0, 40), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
+          .returning('id')
+      )[0].id
+    // Name rules had put the spot's first volume into a season release with another episode.
+    const season = await insertRelease('s', 'zxcvvbgnhgjytygf', 'stem')
+    await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: season, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
+    await file({ filename: 'other.episode.rar', release_id: season, segments: [{ number: 1, id: 'other-1@x', size: 700 }] })
+    const spotFile = { id: null, filename: 'x.nzb', poster: 'Bart', groups: [], date: now.getTime(), first_message_id: 'spot1@spot.net' }
+
+    const result = (await NzbImportService.import(spotFile, XML, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', moveContainer: false, trusted: true }))!
+    assert.notEqual(result.releaseId, Number(season))
+    const own = await db.from('releases').where('id', result.releaseId).first()
+    assert.include(own, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', file_count: 2 })
+    const left = await db.from('files').where('release_id', season).select('filename')
+    assert.deepEqual(left.map((f) => f.filename), ['other.episode.rar'])
+
+    // A par2 or archive name no longer replaces a spot's title.
+    const { isBetterName } = await import('#services/NameService')
+    assert.isTrue(isBetterName('zxcvvbgnhgjytygf', 'Better.Name.S03E285'))
+
+    // Another spot for the same files (a repost) joins the first one's release instead of taking them.
+    const repost = { ...spotFile, first_message_id: 'spot2@spot.net' }
+    const second = (await NzbImportService.import(repost, XML, { name: 'Repost', source: 'spot', moveContainer: false, trusted: true }))!
+    assert.equal(second.releaseId, result.releaseId)
+    assert.equal(second.releaseName, 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264')
+  })
 })

tests/functional/random_names.spec.ts

@@ -0,0 +1,27 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import { looksRandom } from '#services/ArchiveService'
+
+// Names seen on homelab (LOG.md 2026-10-01 (3)).
+const RANDOM = [
+  '5d4063b423f57e1eadda1068b0091409', 'mOzyyxCyYUjAbCdEf12', '6327d9fa2d6acf3e9a61fb053359d5e7.1790574045',
+  'e63afdb4-6a1f-417d-aa1b-b14c0ea1b81d', '1650777167', '83111763 n', 'prvjr7hapd0i1ny14pz4', '8huowjcdamwbetufib9f',
+  '071vl0l120ndtjkjkuh34a2g6aqsak750f55dkudeaea8s4bivjm1l32fd3uduk0dab2t82hp553i29nfjvjt3redp', 'wHJ7s71sAGE', 'EmFqTfL8rxV',
+  'VIDEO_TS', 'Sample', 'CD1', 'Disc 2', 'Subs',
+]
+const READABLE = [
+  'Our.America.with.Lisa.Ling.2011.S01E01.Faith.Healers.1080p.DSCP.WEB-DL.DDP2.0.H.264-FUZEER', '00-graham_barham-club_country',
+  '1790274935_nicovideo_jp_watch_sm26602185', 'StarWars1977', 'nyuu_live_roundtrip', 'Makaja_Gonzales-Freestyle-WEB-2021-XTC_iNT',
+  'The Tom and Jerry Show (2014) S02E13 - Slap Happy Birthday', 'Bondgenoten', 'S01E02', '2026', 'Samples.Of.Love.2020.1080p', 'Subs.and.Dubs',
+]
+
+test.group('random names', () => {
+  test('looksRandom and SQL is_readable_name agree', async ({ assert }) => {
+    for (const name of [...RANDOM, ...READABLE]) {
+      const { rows } = await db.rawQuery('SELECT is_readable_name(?) AS readable', [name])
+      const random = RANDOM.includes(name)
+      assert.equal(looksRandom(name), random, `looksRandom(${name})`)
+      assert.equal(rows[0].readable, !random, `is_readable_name(${name})`)
+    }
+  })
+})

tests/functional/release_merge.spec.ts

@@ -0,0 +1,42 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+
+let keys = 0
+async function postRelease(name: string, files: string[], poster: string) {
+  const now = new Date()
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `merge${++keys}`.padEnd(40, 'm'), name, source: 'post', poster, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  await FileStore.insert(
+    files.map((f, i) => ({ filename: f, poster, date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: '[]',
+      segments: [{ number: 1, id: `${name}-${poster}-${i}@x`, size: 10 }], created_at: now, updated_at: now }))
+  )
+  return Number(r.id)
+}
+
+test.group('Split uploads', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('merges one upload posted under a poster per volume, not reposts or other uploads', async ({ assert }) => {
+    const name = 'Masters.of.the.Universe.1987.Theatrical.Cut.2160p.BDRip-GRP'
+    const a = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part001.rar'], 'niSSYNy5zOW2VX@VMuLu_e7Gr.bIx')
+    const b = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part002.rar'], 'MgrZzFLB3D@MeK4_gntYXzh9g.LBQ')
+    const c = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part003.rar'], 'qNGVviOMN2AJrSzeXFZqvW@VzavS9GO.Y0b')
+    // A repost: the same file names again.
+    const repostName = 'Some.Repost.2026.1080p-GRP'
+    const r1 = await postRelease(repostName, ['abc.part01.rar'], 'p1@x')
+    const r2 = await postRelease(repostName, ['abc.part01.rar'], 'p2@x')
+
+    assert.deepEqual(await ReleaseMergeService.mergeSplitPosts({ dryRun: true }), { names: 1, releases: 2 })
+    await ReleaseMergeService.mergeSplitPosts()
+    assert.equal((await db.from('releases').where('id', a).first()).file_count, 3)
+    assert.equal((await db.from('releases').where('id', b).first()).file_count, 0)
+    assert.equal((await db.from('releases').where('id', c).first()).file_count, 0)
+    assert.equal((await db.from('releases').where('id', r1).first()).file_count, 1)
+    assert.equal((await db.from('releases').where('id', r2).first()).file_count, 1)
+  })
+})

tests/functional/search_text.spec.ts

@@ -0,0 +1,41 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import SearchService, { searchNormalize } from '#services/SearchService'
+
+test.group('Search text', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('normalises like SQL does', async ({ assert }) => {
+    assert.equal(searchNormalize(' The.Block.AU_S22E35 - 1080p '), 'the block au s22e35 1080p')
+    const { rows } = await db.rawQuery(`SELECT search_normalize(' The.Block.AU_S22E35 - 1080p ') AS n`)
+    assert.equal(rows[0].n, 'the block au s22e35 1080p')
+  })
+
+  test('follows files and renames, only for visible releases', async ({ assert }) => {
+    const now = new Date()
+    const [r] = await db
+      .table('releases')
+      .insert({ release_key: 's'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7Rt', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+      .returning('*')
+    assert.isNull(r.search_text)
+
+    // A readable file name that the release name lacks joins the search text.
+    await FileStore.insert([
+      { filename: 'Some.Documentary.2026.1080p.WEB.part01.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
+        groups: '[]', segments: [{ number: 1, id: 'st1@x', size: 10 }], created_at: now, updated_at: now },
+      { filename: 'Q7wErTyUiOpAsDfGhJkL9.part02.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
+        groups: '[]', segments: [{ number: 1, id: 'st2@x', size: 10 }], created_at: now, updated_at: now },
+    ])
+    let row = await db.from('releases').where('id', r.id).first()
+    assert.equal(row.search_text, 'xq8r2kdlm3zw91pqa7rt some documentary 2026 1080p web')
+    const found = await SearchService.search({ q: 'documentary 2026.1080p' } as any)
+    assert.isTrue(found.releases.some((x: any) => x.id === Number(r.id)))
+
+    // A rename recomputes it.
+    await db.from('releases').where('id', r.id).update({ name: 'Some.Documentary.2026.1080p.WEB-GRP' })
+    row = await db.from('releases').where('id', r.id).first()
+    assert.equal(row.search_text, 'some documentary 2026 1080p web grp some documentary 2026 1080p web')
+  })
+})

tests/functional/sfv.spec.ts

@@ -0,0 +1,59 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
+import { parseSfv } from '#lib/content/index'
+
+const POSTER = 'Snazzy Poster <me@localhost.localdomain>'
+
+let keys = 0
+async function release(name: string, source = 'stem') {
+  const now = new Date()
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `sfv${++keys}`.padEnd(40, 'x'), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  return Number(r.id)
+}
+
+async function file(name: string, releaseId: number, poster = POSTER) {
+  const [{ id }] = await FileStore.insert([
+    { filename: name, poster, date: Date.now(), parts: 1, size: 10, release_id: releaseId, groups: '[]',
+      segments: [{ number: 1, id: `${name}@x`, size: 10 }], created_at: new Date(), updated_at: new Date() },
+  ])
+  return db.from('files').where('id', id).first()
+}
+
+test.group('SFV grouping', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('parses SFV listings', ({ assert }) => {
+    assert.deepEqual(parseSfv('; made by x\r\n01-a-b-4066667a.flac 4066667A\r\n"02 c.flac" 5058F120\r\n\r\n'), [
+      { name: '01-a-b-4066667a.flac', crc: '4066667a' },
+      { name: '02 c.flac', crc: '5058f120' },
+    ])
+  })
+
+  test("gathers an album's tracks into the SFV's release, named after the album; later tracks join too", async ({ assert }) => {
+    const album = await release('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026')
+    const sfv = await file('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026.sfv', album)
+    const track1 = await release('01-graham_barham-country_music-4066667a')
+    await file('01-graham_barham-country_music-4066667a.flac', track1)
+    // Another poster's file with the same name stays.
+    const other = await release('01-graham_barham-country_music-4066667a', 'stem')
+    const foreign = await file('01-graham_barham-country_music-4066667a.flac', other, 'someone@else')
+
+    const listing = '01-graham_barham-country_music-4066667a.flac 4066667a\n02-graham_barham-sippin-5058f120.flac 5058f120\n'
+    assert.equal(await SfvService.attach(sfv, listing), 1)
+    const renamed = await db.from('releases').where('id', album).first()
+    assert.equal(renamed.name, 'graham_barham-club_country_(after_party)-16bit-web-flac-2026')
+    assert.equal((await db.from('files').where('id', foreign.id).first()).release_id, other)
+
+    // Track 2 completes after the SFV was read.
+    const track2 = await release('02-graham_barham-sippin-5058f120')
+    const late = await file('02-graham_barham-sippin-5058f120.flac', track2)
+    assert.equal(await SfvService.claim({ id: late.id, name: late.filename, poster: late.poster, date: Number(late.date), release_id: track2 }), album)
+    assert.equal((await db.from('releases').where('id', album).first()).file_count, 3)
+  })
+})

tests/functional/spot_listing.spec.ts

@@ -0,0 +1,109 @@
+import { test } from '@japa/runner'
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import SpotService, { isDummyNzb } from '#services/SpotService'
+import SearchService from '#services/SearchService'
+
+test.group('SpotService listings', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('old verified spots become visible releases without files', async ({ assert }) => {
+    const now = new Date()
+    const old = new Date('2019-05-01T12:00:00Z')
+    const spot = {
+      message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
+      poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
+    }
+    await db.table('spots').insert(spot)
+    const [claimed] = await SpotService.claim(10)
+    assert.isTrue(SpotService.isListable(claimed))
+    assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
+    assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
+
+    assert.equal(await SpotService.list([claimed]), 1)
+    const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
+    assert.equal(row.status, 'listed')
+    const release = await db.from('releases').where('id', row.release_id).first()
+    assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
+    assert.equal(Number(release.size), 1_500_000_000)
+    // The key an import of this spot's NZB uses, so it fills this release.
+    assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
+
+    const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
+    assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
+  })
+
+  test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
+    // A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
+    const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
+    const overviews = Array.from({ length: 6000 }, (_, i) => [
+      i + 1,
+      { 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
+    ]) as any
+    assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
+    const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
+    assert.equal(Number(count), 6000)
+  })
+
+  test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
+    const spot = {
+      message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
+      poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
+      created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
+      description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
+      details_at: new Date(),
+    }
+    await db.table('spots').insert(spot)
+    const [claimed] = await SpotService.claim(10)
+    await SpotService.list([claimed])
+    const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
+    const release = (await SearchService.release(Number(releaseId)))!
+    assert.equal(release.spot!.description, 'Plot[br]more')
+    assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
+    assert.equal(release.spot!.categories.head, 'Image')
+    assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
+  })
+
+  test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
+    const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
+    assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
+    assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
+    // Text-only spots announce the dummy's own size.
+    assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
+  })
+})
+
+test.group('SpotService passwords', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
+    const now = new Date()
+    await db.table('spots').insert({
+      message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
+      key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
+      description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
+    })
+    const [claimed] = await SpotService.claim(10)
+    await SpotService.list([claimed])
+    assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
+    const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
+    await db.from('releases').where('id', releaseId).update({ encrypted: true })
+    const release = (await SearchService.release(Number(releaseId)))!
+    assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
+
+    // An NZB's own password is not replaced.
+    await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
+    await SpotService.applyPassword('pw1@spot.net')
+    assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
+  })
+
+  test('password states', async ({ assert }) => {
+    const { passwordState } = await import('#services/SearchService')
+    assert.equal(passwordState(true, false), 'missing')
+    assert.equal(passwordState(true, true), 'known')
+    assert.equal(passwordState(null, true), 'known')
+    assert.equal(passwordState(false, false), 'none')
+    assert.equal(passwordState(null, false), 'none')
+  })
+})

tests/functional/staging.spec.ts

@@ -0,0 +1,70 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import StagingService from '#services/StagingService'
+import IngestService, { segmentFromHeader } from '#services/IngestService'
+
+const segment = (name: string, k: number, n: number, id = `${name}-${k}@x`) =>
+  segmentFromHeader(
+    { 'subject': `"${name}" yEnc (${k}/${n})`, 'from': 'poster@x', 'message-id': `<${id}>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
+    'alt.binaries.test',
+    k
+  )!
+
+async function binary(name: string) {
+  return db.from('binaries').where('name', name).first()
+}
+
+test.group('Staging', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('a lone segment waits; its second segment brings both in', async ({ assert }) => {
+    let r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
+    assert.equal(r.staged, 1)
+    assert.lengthOf(r.ingest, 0)
+    await IngestService.ingest(r.ingest)
+    assert.isNull(await binary('rnd1.bin'))
+
+    // The same article again (a re-fetched range) is no second segment.
+    r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
+    assert.lengthOf(r.ingest, 0)
+
+    // Nor is the same article crossposted to another group (other group, other article number).
+    const crosspost = { ...segment('rnd1.bin', 3, 50), groups: ['alt.binaries.boneless'], articleNumber: 160154327571 }
+    r = await StagingService.filter([crosspost])
+    assert.lengthOf(r.ingest, 0)
+    r = await StagingService.filter([segment('dup.bin', 2, 9), { ...segment('dup.bin', 2, 9), groups: ['alt.binaries.boneless'] }])
+    assert.equal(r.staged, 1)
+
+    r = await StagingService.filter([segment('rnd1.bin', 4, 50)])
+    assert.deepEqual(r.ingest.map((s) => s.number).sort(), [3, 4])
+    await IngestService.ingest(r.ingest)
+    const b = await binary('rnd1.bin')
+    assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 2)
+    assert.lengthOf(await db.from('staged_segments').where('file_key', segment('rnd1.bin', 3, 50).fileKey), 0)
+
+    // Later segments of a known file go straight in.
+    r = await StagingService.filter([segment('rnd1.bin', 9, 50)])
+    assert.lengthOf(r.ingest, 1)
+  })
+
+  test('single-part files and files with two segments in the batch go straight in; staged ones join them', async ({ assert }) => {
+    // Part 1 arrived alone at the end of the previous batch.
+    await StagingService.filter([segment('real.part01.rar', 1, 3)])
+    const r = await StagingService.filter([segment('real.part01.rar', 2, 3), segment('real.part01.rar', 3, 3), segment('one.nfo', 1, 1)])
+    assert.equal(r.staged, 0)
+    assert.sameMembers(r.ingest.map((s) => `${s.name}:${s.number}`), ['real.part01.rar:1', 'real.part01.rar:2', 'real.part01.rar:3', 'one.nfo:1'])
+  })
+
+  test('GC adopts segments whose file got a binary, and drops old ones', async ({ assert }) => {
+    await StagingService.filter([segment('late.bin', 7, 10)])
+    await StagingService.filter([segment('old.bin', 2, 10)])
+    await db.from('staged_segments').where('file_key', segment('old.bin', 2, 10).fileKey).update({ staged_at: new Date(Date.now() - 3 * 3600_000) })
+    // Another job created the binary meanwhile.
+    await IngestService.ingest([segment('late.bin', 1, 10), segment('late.bin', 2, 10)])
+    const result = await StagingService.sweep(2 * 3600, Date.now() + 10_000)
+    assert.deepEqual(result, { adopted: 1, expired: 1 })
+    const b = await binary('late.bin')
+    assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 3)
+  })
+})

tests/functional/web_api.spec.ts

@@ -1,5 +1,6 @@
 import { test } from '@japa/runner'
 import db from '@adonisjs/lucid/services/db'
+import FileStore from '#services/FileStore'
 import testUtils from '@adonisjs/core/services/test_utils'
 
 async function seed() {
@@ -8,12 +9,12 @@ async function seed() {
     .table('releases')
     .insert({ release_key: 'w'.repeat(40), name: 'Webtest.Show.S01E02.1080p', source: 'par2', groups: '["alt.binaries.test"]', posted_at: now, first_seen_at: now, last_seen_at: now, contents: JSON.stringify([{ name: 'Webtest.Show.S01E02.1080p/episode.mkv', size: 1000 }]) })
     .returning('id')
-  await db.table('files').insert({
+  await FileStore.insert([{
     filename: 'Xq8r2KdLm3Zw91PqA7webtest', real_name: 'Webtest.Show.S01E02.1080p.mkv', poster: 'p@example.invalid',
     date: now.getTime(), parts: 2, size: 1400, release_id: release.id, groups: '["alt.binaries.test"]',
-    message_ids: JSON.stringify({ '1': { id: 'one@test', size: 700 }, '2': { id: 'two@test', size: 700 } }),
+    segments: [{ number: 1, id: 'one@test', size: 700 }, { number: 2, id: 'two@test', size: 700 }],
     created_at: now, updated_at: now,
-  })
+  }])
   return Number(release.id)
 }
 
@@ -67,21 +68,21 @@ test.group('Web API', (group) => {
     const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete
     const file = (releaseId: number, name: string) => ({
       filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
-      groups: '[]', message_ids: '{}', created_at: now, updated_at: now,
+      groups: '[]', segments: [], created_at: now, updated_at: now,
     })
 
     // Counter rule: [i/M] announced 2 files.
     const [counted] = await db.table('releases').insert({ release_key: 'c'.repeat(40), name: 'Counted', groups: '[]', file_total: 2, first_seen_at: now, last_seen_at: now }).returning('id')
-    await db.table('files').insert(file(counted.id, 'a.rar'))
+    await FileStore.insert([file(counted.id, 'a.rar')])
     assert.isFalse(await complete(counted.id))
-    await db.table('files').insert(file(counted.id, 'b.rar'))
+    await FileStore.insert([file(counted.id, 'b.rar')])
     assert.isTrue(await complete(counted.id))
 
     // par2 rule: the listed file becomes present once a file of its exact size is there.
     const [listed] = await db.table('releases').insert({ release_key: 'l'.repeat(40), name: 'Listed', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id')
     await db.table('release_files').insert({ release_id: listed.id, par2_file_id: '1'.repeat(32), name: 'Listed.mkv', size: 5_000_000, md5_16k: '0'.repeat(32) })
     assert.isFalse(await complete(listed.id))
-    const [f] = await db.table('files').insert(file(listed.id, 'Xq8r2KdLm3Zw91PqA7')).returning('id')
+    const [f] = await FileStore.insert([file(listed.id, 'Xq8r2KdLm3Zw91PqA7')])
     const [b] = await db.table('binaries').insert({ file_key: 'z'.repeat(40), poster: 'p', subject: '"x" yEnc', name: 'x', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, file_id: f.id }).returning('id')
     assert.isFalse(await complete(listed.id))
     await db.from('binaries').where('id', b.id).update({ size: 5_000_000 })
@@ -94,8 +95,8 @@ test.group('Web API', (group) => {
       .insert({ release_key: 'p'.repeat(40), name: 'Webtest.Par2only.S01E01.1080p.WEB-DL-GRP', groups: '[]', first_seen_at: now, last_seen_at: now })
       .returning('id')
     await db.table('release_files').insert({ release_id: release.id, par2_file_id: '2'.repeat(32), name: 'Webtest.Par2only.S01E01.mkv', size: 1_000_000_000, md5_16k: '0'.repeat(32) })
-    await db.table('files').insert({ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
-      release_id: release.id, groups: '[]', message_ids: '{}', created_at: now, updated_at: now })
+    await FileStore.insert([{ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
+      release_id: release.id, groups: '[]', segments: [], created_at: now, updated_at: now }])
 
     const hidden = await client.get('/api/v1/releases').qs({ q: 'webtest par2only' })
     assert.notInclude(hidden.body().releases.map((r: any) => r.id), Number(release.id))

tests/functional/yenc_name.spec.ts

@@ -0,0 +1,40 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import FileStore from '#services/FileStore'
+import ContentService from '#services/ContentService'
+
+let keys = 0
+async function counterFile(name: string) {
+  const now = new Date()
+  const [r] = await db
+    .table('releases')
+    .insert({ release_key: `yenc${++keys}`.padEnd(40, 'y'), name, source: 'weak-stem', groups: '[]', first_seen_at: now, last_seen_at: now })
+    .returning('id')
+  const [{ id }] = await FileStore.insert([
+    { filename: name, poster: 'bob@home.com', date: now.getTime(), parts: 2, size: 10, release_id: r.id, groups: '["alt.binaries.boneless"]',
+      segments: [{ number: 1, id: `${name}@nyuu`, size: 10 }], created_at: now, updated_at: now },
+  ])
+  return db.from('files').where('id', id).first()
+}
+
+test.group('Names from the yEnc header', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('random-named files take the yEnc name and gather in its release', async ({ assert }) => {
+    const a = await counterFile('P9BSPl59iDfQUVvXFeBVG4MDQaOTufds')
+    const b = await counterFile('Faoq6nlEaVlTzYyNA27sSfFm2KvLHOvD')
+    const nameFromYenc = (ContentService as any).nameFromYenc.bind(ContentService)
+    assert.equal(await nameFromYenc(a, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar'), 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar')
+    await nameFromYenc(b, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part02.rar')
+    const [fa, fb] = await Promise.all([db.from('files').where('id', a.id).first(), db.from('files').where('id', b.id).first()])
+    assert.equal(fa.release_id, fb.release_id)
+    assert.equal((await db.from('releases').where('id', fa.release_id).first()).name, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP')
+
+    // A random yEnc name, or a file that has a real name already, stays as it is.
+    const c = await counterFile('Vg9QflfNLXVMPOWtK4KRJv474DadgTIF')
+    assert.isUndefined(await nameFromYenc(c, null, 'qTcrIl1qTo4Bx6El5c4RtTA4ZSqwd6tz'))
+    const d = await counterFile('Some.Release.part01.rar')
+    assert.isUndefined(await nameFromYenc(d, null, 'Other.Name.part01.rar'))
+  })
+})

tests/unit/content.spec.ts

@@ -30,6 +30,15 @@ test.group('content / sniffContentType', () => {
 })
 
 test.group('yenc / lenientSize', () => {
+  test("recognises yEncBin's encrypted .bin container by its header", ({ assert }) => {
+    // The first 48 bytes of 2141406641.bin (yEncBin@Poster.com).
+    const data = Buffer.from('54f399d117c492696e4dc4ef5b60c11d7f83e92dfdf23450e79370b048123da81e4c8013091b682fb22478daed2ae981', 'hex')
+    assert.equal(sniffContentType(data), 'yencbin')
+    // The second variant (1060238685.bin).
+    assert.equal(sniffContentType(Buffer.from('54f399e917c492696a4e9fb05b60c31d7f83e92df9f16f5086e3c3d9480d66ef1e48d915', 'hex')), 'yencbin')
+    assert.isNull(sniffContentType(Buffer.from('54f399d117c492696e4dc4ef5b60c11e7f83e92d', 'hex')))
+  })
+
   test('decodes a part whose declared size leaves out a header, still checking the CRC', ({ assert }) => {
     const data = Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(16, 7), Buffer.from('ciphertext-here')])
     const declared = data.length - 20

tests/unit/date_search.spec.ts

@@ -0,0 +1,14 @@
+import { test } from '@japa/runner'
+import { articleAtDate } from '#lib/nntp/date_search'
+
+test.group('nntp / articleAtDate', () => {
+  test('finds the article posted at a time, skipping gaps', async ({ assert }) => {
+    // One article a second from t0, articles 1,000,000–1,100,000; 1,040,000–1,040,999 expired.
+    const t0 = Date.UTC(2026, 8, 1)
+    const dateOf = async (n: bigint) => (n >= 1_040_000n && n < 1_041_000n ? null : t0 + Number(n - 1_000_000n) * 1000)
+    const at = await articleAtDate(1_000_000n, 1_100_000n, t0 + 60_000 * 1000, dateOf, 100n)
+    assert.isAtLeast(Number(at), 1_059_900)
+    assert.isAtMost(Number(at), 1_060_000)
+    assert.equal(await articleAtDate(1_000_000n, 1_100_000n, t0 - 1, dateOf, 100n), 1_000_000n)
+  })
+})

tests/unit/delimited_reader.spec.ts

@@ -0,0 +1,52 @@
+import { test } from '@japa/runner'
+import { delimitedReader } from '#lib/nntp/delimited_reader'
+
+const CRLF = Buffer.from('\r\n')
+const END = Buffer.from('\r\n.\r\n')
+
+async function* chunks(...parts: string[]) {
+  for (const part of parts) yield Buffer.from(part, 'latin1')
+}
+
+test.group('nntp / delimitedReader', () => {
+  test('reads an empty multi-line reply (XOVER of a range without articles)', async ({ assert }) => {
+    const reader = delimitedReader(chunks('224 Overview follows\r\n.\r\n', '211 1 2 3 g\r\n'), CRLF)
+    assert.equal((await reader.next()).value!.toString(), '224 Overview follows')
+    assert.equal((await reader.next(END)).value!.toString(), '')
+    assert.equal((await reader.next(CRLF)).value!.toString(), '211 1 2 3 g')
+  })
+
+  test('reads multi-line data, also with the terminator split over chunks', async ({ assert }) => {
+    for (const split of [1, 2, 3, 4, 5, 6, 7]) {
+      const reply = '224 ok\r\nline 1\r\n..dotted\r\nline 3\r\n.\r\n200 next\r\n'
+      const at = reply.indexOf('.\r\n200') - 2 + split - 1
+      const reader = delimitedReader(chunks(reply.slice(0, at), reply.slice(at)), CRLF)
+      assert.equal((await reader.next()).value!.toString(), '224 ok')
+      assert.equal((await reader.next(END)).value!.toString(), 'line 1\r\n..dotted\r\nline 3', `split ${split}`)
+      assert.equal((await reader.next(CRLF)).value!.toString(), '200 next')
+    }
+  })
+
+  test('reads replies byte by byte', async ({ assert }) => {
+    const text = '200 welcome\r\n224 ok\r\na\r\n.\r\n224 ok\r\n.\r\n211 x\r\n'
+    const reader = delimitedReader(chunks(...text.split('')), CRLF)
+    assert.equal((await reader.next()).value!.toString(), '200 welcome')
+    assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
+    assert.equal((await reader.next(END)).value!.toString(), 'a')
+    assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
+    assert.equal((await reader.next(END)).value!.toString(), '')
+    assert.equal((await reader.next(CRLF)).value!.toString(), '211 x')
+  })
+
+  test('keeps binary data intact', async ({ assert }) => {
+    const body = Buffer.from([0, 13, 255, 10, 46, 13, 46, 10, 13])
+    const source = (async function* () {
+      yield Buffer.from('222 body\r\n')
+      yield body
+      yield Buffer.from('\r\n.\r\n')
+    })()
+    const reader = delimitedReader(source, CRLF)
+    await reader.next()
+    assert.deepEqual((await reader.next(END)).value, body)
+  })
+})

tests/unit/nicovideo.spec.ts

@@ -0,0 +1,46 @@
+import { test } from '@japa/runner'
+import { randomBytes } from 'node:crypto'
+import { zstdCompressSync, constants } from 'node:zlib'
+import { NICOVIDEO_FILE, nicovideoInfo, nicovideoReleaseName, zstdPrefix } from '#lib/content/index'
+
+function tarEntry(name: string, data: Buffer): Buffer {
+  const header = Buffer.alloc(512)
+  header.write(name, 0, 'utf8')
+  header.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 'latin1')
+  header.write('ustar\0', 257, 'latin1')
+  return Buffer.concat([header, data, Buffer.alloc((512 - (data.length % 512)) % 512)])
+}
+
+test.group('nicovideo captures', () => {
+  test('reads the title from the start of a capture, even a truncated one', async ({ assert }) => {
+    const dir = '1790720489_nicovideo_jp_watch_sm27965722'
+    const response = {
+      data: {
+        metadata: { jsonLds: [{ author: { name: 'まふまふ' } }] },
+        response: {
+          video: { id: 'sm27965722', title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】', registeredAt: '2016-01-08T20:00:00+09:00' },
+          owner: { nickname: 'まふまふ' },
+          tag: { items: [{ name: '音楽' }, { name: 'After_the_Rain' }] },
+        },
+      },
+    }
+    const tar = Buffer.concat([
+      tarEntry(`${dir}/`, Buffer.alloc(0)),
+      tarEntry(`${dir}/server_response.metadata`, Buffer.from(JSON.stringify(response, null, 2))),
+      tarEntry(`${dir}/01.cmfa`, randomBytes(300_000)),
+    ])
+    const compressed = zstdCompressSync(tar, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
+    // Only the first part of the stream, as in the file's first article.
+    const info = nicovideoInfo(await zstdPrefix(compressed.subarray(0, Math.floor(compressed.length * 0.6))))!
+    assert.deepEqual(info, {
+      id: 'sm27965722',
+      title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】',
+      owner: 'まふまふ',
+      registeredAt: '2016-01-08T20:00:00+09:00',
+      tags: ['音楽', 'After_the_Rain'],
+    })
+    assert.equal(nicovideoReleaseName(info, 'sm1'), '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】 - まふまふ (nicovideo sm27965722)')
+    assert.equal(NICOVIDEO_FILE.exec(`${dir}.tar.zst`)![1], 'sm27965722')
+    assert.isNull(nicovideoInfo(await zstdPrefix(Buffer.from('not zstd'))))
+  })
+})

tests/unit/nzb.spec.ts

@@ -10,7 +10,7 @@ test.group('nzb / buildNzb', () => {
           poster: 'poster@example.invalid',
           date: 1_790_000_000_000,
           groups: ['alt.binaries.test'],
-          segments: { '2': { id: 'b@x', size: 700 }, '1': { id: 'a@x', size: 800 } },
+          segments: [{ number: 2, id: 'b@x', size: 700 }, { number: 1, id: 'a@x', size: 800 }],
         },
       ],
       { title: 'Some.Show.S01E01' }
@@ -53,3 +53,15 @@ test.group('nzb / parseNzb', () => {
     assert.isNull(parseNzb('<html>not an nzb</html>'))
   })
 })
+
+test.group('nzb / stored segment lists', () => {
+  test('pack and unpack round-trip, in order, also with gaps in the numbering', async ({ assert }) => {
+    const { packSegments, unpackSegments } = await import('#lib/nzb/segments')
+    const sequential = Array.from({ length: 300 }, (_, i) => ({ number: i + 1, id: `Ab${i}Cd-1790600369${i}@nyuu`, size: i === 299 ? 1234 : 740000 }))
+    assert.deepEqual(unpackSegments(packSegments([...sequential].reverse())), sequential)
+    const gaps = [{ number: 3, id: 'c@x', size: 3 }, { number: 1, id: 'a@x', size: 1 }, { number: 7, id: 'g@x', size: 7 }]
+    assert.deepEqual(unpackSegments(packSegments(gaps)), [gaps[1], gaps[0], gaps[2]])
+    assert.deepEqual(unpackSegments(packSegments([])), [])
+    assert.isBelow(packSegments(sequential).length / sequential.length, 30)
+  })
+})

tests/unit/sevenzip.spec.ts

@@ -0,0 +1,71 @@
+import { test } from '@japa/runner'
+import {
+  checkHeader,
+  listingStart,
+  parseBsdtarListing,
+  parseSevenZipStart,
+  readNumber,
+  sevenZipVolume,
+} from '#lib/sevenzip/index'
+
+// `7z a tiny.7z Tiny.Release` (Tiny.Release/a.txt): an LZMA-encoded header.
+const ENCODED = Buffer.from(
+  'N3q8ryccAATeIVLCfQAAAAAAAAAhAAAAAAAAAGkLW/MBAAtoZWxsbyB3b3JsZAoAAACBMweuD87ysgwHyEN/QbH6/duIttdfpOXRhZbi6t7tycGOUDyW1l2aTE7xmuZ/NAXKgg599VxKYAF1XImsfT9LzHQKaEH6PEZambLnwJM62mXXEJIpaYvhCrHFeQv+jLnYjgJMfwZEcAAAABcGEAEJbQAHCwEAASMDAQEFXQAQAAAMgJYKASTou9wAAA==',
+  'base64'
+)
+// The same with -mhc=off: a plain header.
+const PLAIN = Buffer.from(
+  'N3q8ryccAASpRwRsEAAAAAAAAACWAAAAAAAAACTou9wBAAtoZWxsbyB3b3JsZAoAAQQGAAEJEAAHCwEAASEhAQAMDAAICgEtOwivAAAFAg4BgBkJAAAAAAAAAAAAEUEAVABpAG4AeQAuAFIAZQBsAGUAYQBzAGUAAABUAGkAbgB5AC4AUgBlAGwAZQBhAHMAZQAvAGEALgB0AHgAdAAAABkCAAAUEgEA/QYYLBBR3QH9BhgsEFHdARUKAQAQgP1BIIC0gQAA',
+  'base64'
+)
+
+test.group('sevenzip', () => {
+  test('finds the header at the end from the signature header', ({ assert }) => {
+    const start = parseSevenZipStart(ENCODED)!
+    assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })
+    const header = ENCODED.subarray(start.headerStart, start.archiveSize)
+    assert.isTrue(checkHeader(start, header))
+    assert.isFalse(checkHeader(start, Buffer.alloc(33)))
+    // Encoded: the packed header right after the file data (packPos 16) is needed too.
+    assert.equal(listingStart(start, header), 48)
+
+    const plain = parseSevenZipStart(PLAIN)!
+    assert.equal(plain.archiveSize, PLAIN.length)
+    assert.equal(listingStart(plain, PLAIN.subarray(plain.headerStart)), plain.headerStart)
+  })
+
+  test('rejects other data and a damaged signature header', ({ assert }) => {
+    assert.isNull(parseSevenZipStart(Buffer.from('Rar!\x1a\x07\x01\x00'.padEnd(40, '\0'), 'latin1')))
+    const damaged = Buffer.from(ENCODED)
+    damaged[14] ^= 1
+    assert.isNull(parseSevenZipStart(damaged))
+  })
+
+  test('reads 7z numbers', ({ assert }) => {
+    assert.deepEqual(readNumber(Buffer.from([0x10]), 0), [16, 1])
+    assert.deepEqual(readNumber(Buffer.from([0x81, 0x02]), 0), [0x102, 2])
+    assert.deepEqual(readNumber(Buffer.from([0xc0, 0x34, 0x12]), 0), [0x1234, 3])
+    assert.deepEqual(readNumber(Buffer.from([0xff, 1, 0, 0, 0, 0, 0, 0, 0]), 0), [1, 9])
+  })
+
+  test('parses bsdtar listings', ({ assert }) => {
+    const out = [
+      'drwxrwxr-x  0 0      0           0 Sep 30 19:13 Some.Release.2026/',
+      '-rw-rw-r--  0 0      0     3000000 Sep 30 19:13 Some.Release.2026/movie.mkv',
+      '-rw-rw-r--  0 0      0           3 Jan  1  2019 Some.Release.2026/some info.nfo',
+      '',
+    ].join('\n')
+    assert.deepEqual(parseBsdtarListing(out), [
+      { name: 'Some.Release.2026', size: 0, directory: true },
+      { name: 'Some.Release.2026/movie.mkv', size: 3000000, directory: false },
+      { name: 'Some.Release.2026/some info.nfo', size: 3, directory: false },
+    ])
+  })
+
+  test('numbers volumes', ({ assert }) => {
+    assert.equal(sevenZipVolume('x.7z.001'), 1)
+    assert.equal(sevenZipVolume('X.7Z.012'), 12)
+    assert.equal(sevenZipVolume('x.7z'), 0)
+    assert.isNull(sevenZipVolume('x.part01.rar'))
+  })
+})

tests/unit/spotnet.spec.ts

@@ -1,6 +1,15 @@
 import { test } from '@japa/runner'
 import { deflateRawSync } from 'node:zlib'
-import { decodeSpotBinary, parseSpotArticleHeaders, parseSpotHeader, parseSpotXml, verifySpotHeader } from '#lib/spotnet/index'
+import {
+  decodeSpotBinary,
+  decodeSpotText,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  spotCategoryLabels,
+  verifySpotHeader,
+} from '#lib/spotnet/index'
+import { ubbToText } from '#lib/newznab/xml'
 
 // A real spot from free.pt (self-signed, key 7), 2026-09-28.
 const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "<jWCbaAa4f1A7LC6agmOEx@spot.net>", "date": "Mon, 28 Sep 2026 18:24:45 GMT"}
@@ -44,6 +53,62 @@ test.group('spotnet / xml and nzb', () => {
     assert.deepEqual(spot.subcats, ['01a09', '01z00'])
   })
 
+  test('reads the description, tag and image', ({ assert }) => {
+    const spot = parseSpotXml(
+      `<Spotnet><Posting><Key>7</Key><Tag>Bart</Tag><Title>T</Title>` +
+        `<Description>Lily Tomlin[br][br]If you&#39;re &#8216;in&#8217; [url=https://example.com/a]here[/url]</Description>` +
+        `<Image Width='185' Height='278'><Segment>FYe6l94kQsY7LC6agh5Xn@spot.net</Segment></Image>` +
+        `<Category>01<Sub>01a09</Sub></Category></Posting></Spotnet>`
+    )!
+    assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]')
+    assert.equal(spot.tag, 'Bart')
+    assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 })
+    // Older spots link an image on the web.
+    assert.deepEqual(parseSpotXml('<Posting><Image>http://x.nl/a.jpg</Image></Posting>')!.image, { url: 'http://x.nl/a.jpg' })
+    assert.isNull(parseSpotXml('<Posting><Title>T</Title></Posting>')!.image)
+  })
+
+  test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => {
+    const raw = [
+      'From: vanker',
+      ' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>',
+      'Subject: The postman | The postman |',
+      'X-XML: <SpotNet><Posting><Title>The postman</Title><Description><![CDATA[post voor de lol.<br />Van &quot;The',
+      'X-XML:  postman&quot;]]></Description><Image>http://covertarget.com/si.php?id=1</Image><Category>00</Category></Posting></SpotNet>',
+      'Date: 01 Feb 2010 02:43:16 GMT',
+    ].join('\r\n')
+    const { headers, xml } = parseSpotArticleHeaders(raw)
+    assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>')
+    assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT')
+    const spot = parseSpotXml(xml)!
+    assert.equal(spot.description, 'post voor de lol.<br />Van "The postman"')
+    assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' })
+    assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"')
+  })
+
+  test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => {
+    assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café')
+    assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café')
+  })
+
+  test('names categories as Spotweb does', ({ assert }) => {
+    assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), {
+      head: 'Image',
+      subcats: [
+        { type: 'Type', values: ['Movie'] },
+        { type: 'Format', values: ['x264'] },
+        { type: 'Source', values: ['Retail'] },
+        { type: 'Language', values: ['English subtitles (available)', 'English audio/written'] },
+        { type: 'Genre', values: ['Science Fiction'] },
+      ],
+    })
+    assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] })
+  })
+
+  test('turns UBB markup into plain text', ({ assert }) => {
+    assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)')
+  })
+
   test('decodes an escaped, deflated NZB split over articles', ({ assert }) => {
     const nzb = '<?xml version="1.0"?>\n<nzb>\r\n=\0</nzb>'.repeat(20)
     const escaped = deflateRawSync(Buffer.from(nzb, 'latin1'))
@@ -68,3 +133,33 @@ test.group('spotnet / moderation', () => {
     assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/)
   })
 })
+
+test.group('spotnet / passwords in descriptions', () => {
+  test('finds passwords written in the description or title', async ({ assert }) => {
+    const { passwordFromSpot } = await import('#lib/spotnet/index')
+    assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2')
+    assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R')
+    assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813')
+    assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401')
+    assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo')
+    assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t')
+    assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
+  })
+
+  test('ignores sentences about passwords', async ({ assert }) => {
+    const { passwordFromSpot } = await import('#lib/spotnet/index')
+    for (const description of [
+      'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]',
+      '[b]--> Geen wachtwoord benodigd [/b]',
+      'zitten 2 bestanden in en is met een password beschermd.',
+      'It also removes the need for manual password entry and improves the reliability',
+      'Codeermodus: Hoge kwaliteit (2-Pass VBR)',
+      '29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper',
+      'Wachtwoord: geen',
+      'WW: zie nzb',
+      'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
+    ]) {
+      assert.isNull(passwordFromSpot('T', description), description)
+    }
+  })
+})