Spotnet import, header backfill, staging and counter grouping, faster search

Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/ContentBackfill.ts

@@ -3,6 +3,7 @@ import { CommandOptions } from '@adonisjs/core/types/ace'
 import db from '@adonisjs/lucid/services/db'
 import QueueService from '#services/QueueService'
 import ContentService from '#services/ContentService'
+import { isArchiveStart } from '#lib/content/index'
 
 const BATCH = 1000
 
@@ -28,17 +29,37 @@ export default class ContentBackfill extends BaseCommand {
   @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
   declare archiveNzbs: boolean
 
+  @flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
+  declare sevenZip: boolean
+
+  @flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
+  declare archives: boolean
+
+  @flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
+  declare sfv: boolean
+
+  @flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
+  declare nicovideo: boolean
+
   public async run() {
     let lastId = 0
     let queued = 0
     try {
       while (!this.limit || queued < this.limit) {
         // Files never inspected: no fingerprint on their binary and no archive listing.
-        const query = db
-          .from('files as f')
-          .join('binaries as b', 'b.file_id', 'f.id')
-          .where('f.id', '>', lastId)
-        if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
+        const query = db.from('files as f').where('f.id', '>', lastId)
+        // Files from NZBs (spots) have no binary.
+        if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
+        else query.join('binaries as b', 'b.file_id', 'f.id')
+        if (this.nicovideo) {
+          query.whereNull('f.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
+        } else if (this.sfv) {
+          query.whereNull('f.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
+        } else if (this.archives) {
+          query.whereNull('f.content_type').whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
+        } else if (this.sevenZip) {
+          query.whereNull('f.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
+        } else if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`)
         else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive')
         const files = await query
           .orderBy('f.id')
@@ -48,14 +69,19 @@ export default class ContentBackfill extends BaseCommand {
         lastId = Number(files.at(-1).id)
 
         const jobs = files
-          .filter((f) => ContentService.isCandidate(f.filename))
+          .filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
           .slice(0, this.limit ? this.limit - queued : undefined)
           .map((f) => ({
             name: 'process-collection',
             data: { fileId: Number(f.id) },
             // A new id per run: BullMQ ignores an add() for an id it still knows.
             // Targeted backfills go ahead of whatever is waiting.
-            opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs },
+            opts: {
+              jobId: `content-${f.id}-${Date.now()}`,
+              lifo: this.archiveNzbs || this.sevenZip || this.archives || this.sfv,
+              // niconico captures are many (~190k) and only cosmetic: behind everything else.
+              ...(this.nicovideo ? { priority: 10 } : {}),
+            },
           }))
         if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs)
         queued += jobs.length

commands/DbReindex.ts

@@ -0,0 +1,22 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import IndexMaintenance from '#services/IndexMaintenance'
+
+/** Rebuilds bloated indexes online (gc:scheduler also does this weekly). */
+export default class DbReindex extends BaseCommand {
+  public static commandName = 'db:reindex'
+  public static description = 'Rebuilds B-tree indexes that are mostly empty space (online, REINDEX CONCURRENTLY).'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show which indexes would be rebuilt' })
+  declare dryRun: boolean
+
+  public async run() {
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`${rebuilt.length} indexes ${this.dryRun ? 'would be rebuilt' : `rebuilt, ${Math.round(saved / 1024 / 1024)} MB freed`}.`)
+  }
+}

commands/FilesCleanNames.ts

@@ -0,0 +1,35 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import { unquotedFileName } from '#services/NzbImportService'
+
+/**
+ * Files named after a whole NZB subject without quotes (spot NZBs), e.g.
+ * `Adventure.part0055.rar - [056/175] (1/1)`: renamed to the filename in it.
+ */
+export default class FilesCleanNames extends BaseCommand {
+  public static commandName = 'files:clean-names'
+  public static description = 'Strips subject leftovers (counters, "yEnc") from file names that came from NZBs.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only show what would change' })
+  declare dryRun: boolean
+
+  public async run() {
+    const files = await db
+      .from('files')
+      .whereRaw(`filename ~* '(\\(\\d+/\\d+\\)|yEnc|\\[\\d+/\\d+\\])\\s*$'`)
+      .select('id', 'filename')
+    const changes = files
+      .map((f) => ({ id: Number(f.id), from: f.filename as string, to: unquotedFileName(f.filename) }))
+      .filter((c) => c.to !== c.from)
+    for (const c of changes.slice(0, 5)) this.logger.info(`"${c.from}" → "${c.to}"`)
+    if (!this.dryRun) {
+      for (const c of changes) await db.from('files').where('id', c.id).update({ filename: c.to })
+    }
+    this.logger.info(`${changes.length} of ${files.length} file names ${this.dryRun ? 'would change' : 'cleaned'}.`)
+  }
+}

commands/GcScheduler.ts

@@ -5,6 +5,9 @@ import db from '@adonisjs/lucid/services/db'
 import gcConfig from '#config/gc'
 import QueueService from '#services/QueueService'
 import RedisService from '#services/RedisService'
+import IndexMaintenance from '#services/IndexMaintenance'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+import StagingService from '#services/StagingService'
 import Shutdown from '#services/Shutdown'
 
 const CLEAN_BATCH = 1000
@@ -63,13 +66,20 @@ export default class GcScheduler extends BaseCommand {
     this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
 
     if (!this.dryRun) {
-      const budget = Date.now() + gcConfig.deleteBudgetMs
-      const parts = await this.dropCopiedParts(budget)
-      const single = await this.deleteSingleArticleBinaries(budget)
-      const releases = await this.deleteEmptyReleases(budget)
-      if (parts || single || releases) {
+      // Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
+      const budget = () => Date.now() + gcConfig.deleteBudgetMs
+      const parts = await this.dropCopiedParts(budget())
+      const single = await this.deleteSingleArticleBinaries(budget())
+      const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
+      const releases = await this.deleteEmptyReleases(budget())
+      const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
+      if (staging.adopted || staging.expired) {
+        this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
+      }
+      if (parts || single || unfingerprinted || releases) {
         this.logger.info(
-          `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.`
+          `Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
+            `${releases} empty releases deleted.`
         )
       }
     }
@@ -83,6 +93,23 @@ export default class GcScheduler extends BaseCommand {
         `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
       )
     }
+
+    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep.
+    const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
+
+    await this.rebuildIndexes()
+  }
+
+  /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
+  private async rebuildIndexes() {
+    const key = 'gc:reindex:last'
+    const last = Number((await RedisService.client.get(key)) ?? 0)
+    if (Date.now() - last < gcConfig.reindexIntervalMs) return
+    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
+    if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
+    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
+    this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
   }
 
   /**
@@ -151,10 +178,11 @@ export default class GcScheduler extends BaseCommand {
     let total = 0
     while (Date.now() < budget) {
       const result = await db.rawQuery(
-        `DELETE FROM parts WHERE binary_id IN (
+        // Header-derived parts (no per-part CRC) of binaries that are a file now.
+        `DELETE FROM parts WHERE pcrc32 IS NULL AND binary_id IN (
            SELECT b.id FROM binaries b
-           WHERE b.file_id IS NOT NULL AND b.size IS NULL
-             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id)
+           WHERE b.file_id IS NOT NULL
+             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id AND p.pcrc32 IS NULL)
            LIMIT ?)`,
         [500]
       )
@@ -190,6 +218,29 @@ export default class GcScheduler extends BaseCommand {
     return total
   }
 
+  /**
+   * Completed header-derived binaries without a fingerprint, some hours after
+   * completion: their file exists, their parts are gone, and par2 naming needs
+   * a size and MD5 they don't have. Oldest first, via the last_seen_at index.
+   */
+  private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
+    const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
+    let total = 0
+    while (Date.now() < budget) {
+      const result = await db.rawQuery(
+        `DELETE FROM binaries WHERE id IN (
+           SELECT id FROM binaries
+           WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
+             AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
+           ORDER BY last_seen_at LIMIT ?)`,
+        [cutoff, gcConfig.deleteBatch]
+      )
+      total += result.rowCount
+      if (result.rowCount < gcConfig.deleteBatch) break
+    }
+    return total
+  }
+
   /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
   private async deleteEmptyReleases(budget: number): Promise<number> {
     const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)

commands/GroupAdd.ts

@@ -33,6 +33,9 @@ export default class GroupAdd extends BaseCommand {
   @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
   declare reset: boolean
 
+  @flags.boolean({ description: "Start at the server's oldest article: the group's full history (also for groups indexed before)" })
+  declare fromStart: boolean
+
   public async run() {
     for (const name of this.names) {
       if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
@@ -42,26 +45,30 @@ export default class GroupAdd extends BaseCommand {
       const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
       group.active = !this.disable
       if (this.bodies) group.fetchBodies = true
-      if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
-        // The scheduler continues after last_queued_id.
-        const last = await this.lastArticle(name)
+      if (this.fromStart) {
+        // The scheduler continues after last_queued_id; 0 would mean "new group" to it.
+        const { first } = await this.groupInfo(name)
+        group.lastQueuedId = first > 1n ? first - 1n : 1n
+        group.lastIndexedId = group.lastQueuedId
+      } else if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
+        const { last } = await this.groupInfo(name)
         group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
         group.lastIndexedId = group.lastQueuedId
       }
       await group.save()
       this.logger.info(
         `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
-          (this.backfill !== undefined ? `, starting after article ${group.lastQueuedId}` : '')
+          (this.backfill !== undefined || this.fromStart ? `, starting after article ${group.lastQueuedId}` : '')
       )
     }
-    if (this.backfill !== undefined) await NntpService.shutdown()
+    if (this.backfill !== undefined || this.fromStart) await NntpService.shutdown()
   }
 
-  private async lastArticle(name: string): Promise<bigint> {
+  private async groupInfo(name: string) {
     const conn = await NntpService.acquire()
     let failed = false
     try {
-      return parseGroupResponse((await conn.group(name)).resp).last
+      return parseGroupResponse((await conn.group(name)).resp)
     } catch (error) {
       failed = true
       throw error

commands/GroupBackfill.ts

@@ -0,0 +1,52 @@
+import { BaseCommand, args, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import BackfillService from '#services/BackfillService'
+import { isSpotGroup } from '#services/SpotService'
+
+/**
+ * Indexes a group's older headers (CONCEPT.md D32): from where live
+ * indexing began back to N days ago, header-only and behind live indexing.
+ * index:scheduler queues the ranges; this only sets where to go.
+ */
+export default class GroupBackfill extends BaseCommand {
+  public static commandName = 'group:backfill'
+  public static description = "Indexes groups' older headers (header-only, behind live indexing)."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @args.spread({ description: 'Groups (default: every active binary group)', required: false })
+  declare names?: string[]
+
+  @flags.number({ description: 'How many days back', default: 21 })
+  declare days: number
+
+  @flags.boolean({ description: 'Stop backfilling these groups (ranges already queued still run)' })
+  declare stop: boolean
+
+  public async run() {
+    const names = this.names?.length
+      ? this.names
+      : (await db.from('groups').where('active', true).orderBy('name').select('name')).map((g) => g.name as string).filter((n) => !isSpotGroup(n))
+    try {
+      for (const name of names) {
+        if (this.stop) {
+          await BackfillService.stop(name)
+          this.logger.info(`${name}: backfill stopped.`)
+          continue
+        }
+        const range = await BackfillService.start(name, this.days)
+        this.logger.info(
+          range
+            ? `${name}: backfilling articles ${range.from}–${range.to} (${(range.to - range.from).toLocaleString('en')} articles, ${this.days} days).`
+            : `${name}: nothing to backfill.`
+        )
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+  }
+}

commands/IndexScheduler.ts

@@ -7,6 +7,7 @@ import indexerConfig from '#config/indexer'
 import Shutdown from '#services/Shutdown'
 import FetchRangeService from '#services/FetchRangeService'
 import SplitService from '#services/SplitService'
+import BackfillService from '#services/BackfillService'
 
 export default class IndexScheduler extends BaseCommand {
   public static commandName = 'index:scheduler'
@@ -107,9 +108,19 @@ export default class IndexScheduler extends BaseCommand {
       }
     }
 
+    const backfill = async () => {
+      try {
+        const queued = await BackfillService.tick()
+        if (queued > 0) this.logger.info(`Queued ${queued} backfill range(s).`)
+      } catch (err: any) {
+        this.logger.error(`Error queuing backfill: ${err.message}`)
+      }
+    }
+
     // Run immediately, then wait a full interval after each run so runs never overlap.
     while (!Shutdown.requested) {
       await schedule()
+      await backfill()
       await split()
       await Shutdown.sleep(indexerConfig.intervalMs)
     }

commands/ReleaseMergeSplit.ts

@@ -0,0 +1,21 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import ReleaseMergeService from '#services/ReleaseMergeService'
+
+/** Merges uploads split per poster (gc:scheduler also does this every sweep). */
+export default class ReleaseMergeSplit extends BaseCommand {
+  public static commandName = 'release:merge-split'
+  public static description = 'Merges releases of one upload that was posted under a different poster per file.'
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const { names, releases } = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
+    this.logger.info(`${names} split uploads; ${releases} releases ${this.dryRun ? 'would be' : ''} merged into their first.`)
+  }
+}

commands/SpotPasswords.ts

@@ -0,0 +1,25 @@
+import { BaseCommand } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import SpotService from '#services/SpotService'
+
+/** Passwords from the titles and descriptions of spots read so far, onto their releases. */
+export default class SpotPasswords extends BaseCommand {
+  public static commandName = 'spot:passwords'
+  public static description = "Takes archive passwords from spots' titles and descriptions onto their releases."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  public async run() {
+    const spots = await db
+      .from('spots')
+      .whereNotNull('release_id')
+      .where((q) => q.whereNotNull('description').orWhere('title', 'like', '%{{%}}%'))
+      .select('message_id')
+    let found = 0
+    for (const spot of spots) if (await SpotService.applyPassword(spot.message_id)) found++
+    this.logger.info(`${found} of ${spots.length} spots name a password in their title or description.`)
+  }
+}

commands/SpotReimport.ts

@@ -0,0 +1,51 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import { CommandOptions } from '@adonisjs/core/types/ace'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import SpotService from '#services/SpotService'
+
+/**
+ * Imported spots whose files sit in a release that isn't the spot's own
+ * (another source, or another name): their NZB is read again, which moves
+ * the files into the spot's release under its title (CONCEPT.md D29).
+ */
+export default class SpotReimport extends BaseCommand {
+  public static commandName = 'spot:reimport'
+  public static description = "Moves imported spots' files into their own release, named after the spot."
+
+  public static options: CommandOptions = {
+    startApp: true,
+  }
+
+  @flags.boolean({ description: 'Only count them' })
+  declare dryRun: boolean
+
+  public async run() {
+    const spots = await db
+      .from('spots as s')
+      .join('releases as r', 'r.id', 's.release_id')
+      .where('s.status', 'imported')
+      .where((q) => q.whereNot('r.source', 'spot').orWhereRaw('r.name <> s.title'))
+      .orderBy('s.posted_at', 'desc')
+      .select('s.*', 'r.name as release_name')
+    this.logger.info(`${spots.length} imported spots are in a release that isn't their own.`)
+    if (this.dryRun) return
+    let moved = 0
+    let failed = 0
+    try {
+      for (const spot of spots) {
+        try {
+          const releaseId = await SpotService.reimport(spot)
+          if (releaseId) moved++
+          else failed++
+        } catch (error: any) {
+          failed++
+          this.logger.warning(`${spot.title}: ${error.message}`)
+        }
+      }
+    } finally {
+      await NntpService.shutdown()
+    }
+    this.logger.info(`${moved} spots re-imported, ${failed} without a usable NZB (dummy, gone) or failed.`)
+  }
+}

commands/workers/Collection.ts

@@ -49,6 +49,10 @@ export default class CollectionWorker extends BaseCommand {
         if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
         if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
         if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
+        if (content?.yencName) this.logger.info(`"${file.filename}" is "${content.yencName}" (yEnc name).`)
+        if (content?.nicovideo) this.logger.info(`"${file.filename}" → "${content.nicovideo}".`)
+        if (content?.sfvMoved) this.logger.info(`SFV "${file.filename}" moved ${content.sfvMoved} files into its release.`)
+        if (content?.contentType === 'yencbin') this.logger.info(`"${file.filename}" is a yEncBin encrypted container.`)
         if (content?.nzb) {
           const n = content.nzb
           this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)

commands/workers/Fetch.ts

@@ -12,6 +12,7 @@ import FetchRangeService from '#services/FetchRangeService'
 import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
 import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
 import SpotService, { isSpotGroup } from '#services/SpotService'
+import StagingService from '#services/StagingService'
 
 export default class FetchWorker extends BaseCommand {
   public static commandName = 'worker:fetch'
@@ -36,8 +37,10 @@ export default class FetchWorker extends BaseCommand {
       let failed = false
       try {
         conn = await pool.acquire()
-        await conn.group(groupName)
-        overviews = (await conn.xover(startId, endId)).overviews
+        await pool.timed(conn.group(groupName), `GROUP ${groupName}`)
+        const reply = await pool.timed<any>(conn.xover(startId, endId), `XOVER ${groupName} ${startId}-${endId}`)
+        // An empty reply parses as one line without fields ([null, {}]).
+        overviews = reply.overviews.filter(([number, header]: [number | null, Record<string, string>]) => number !== null && header?.['message-id'])
       } catch (error: any) {
         failed = true
         this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
@@ -56,7 +59,8 @@ export default class FetchWorker extends BaseCommand {
       }
 
       const group = await db.from('groups').where('name', groupName).first()
-      const fetchBodies = Boolean(group?.fetch_bodies)
+      // Backfill is header-only: old bodies would be far more than the link carries (CONCEPT.md D32).
+      const fetchBodies = Boolean(group?.fetch_bodies) && !job.data.backfill
 
       const segments: IngestSegment[] = []
       const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
@@ -72,7 +76,11 @@ export default class FetchWorker extends BaseCommand {
         else obfuscated.push({ articleNumber, header })
       }
 
-      const completed = await IngestService.ingest(segments)
+      // Lone segments of new files wait in staging until their file proves real (D35).
+      const { ingest, staged } = StagingService.enabled(groupName)
+        ? await StagingService.filter(segments, { counters: StagingService.countersEnabled(groupName) })
+        : { ingest: segments, staged: 0 }
+      const completed = await IngestService.ingest(ingest)
       const deferredFiles = await DeferredService.record(groupName, deferred)
 
       if (obfuscated.length > 0) {
@@ -82,7 +90,7 @@ export default class FetchWorker extends BaseCommand {
       if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
 
       this.logger.info(
-        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${segments.length} stored, ` +
+        `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${ingest.length} stored${staged ? `, ${staged} staged` : ''}, ` +
           `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
           `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
       )

commands/workers/File.ts

@@ -2,6 +2,9 @@ import { BaseCommand } from '@adonisjs/core/ace'
 import { CommandOptions } from '@adonisjs/core/types/ace'
 import { Worker } from 'bullmq'
 import db from '@adonisjs/lucid/services/db'
+import env from '#start/env'
+import FileStore from '#services/FileStore'
+import SfvService from '#services/SfvService'
 import queueConfig from '#config/queue'
 import Shutdown from '#services/Shutdown'
 import QueueService from '#services/QueueService'
@@ -42,15 +45,10 @@ export default class FileWorker extends BaseCommand {
         throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`)
       }
 
-      const messageIds = Object.fromEntries(
-        parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }])
-      )
+      const segments = parts.map((part) => ({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) }))
 
       // Already known as a file, e.g. added from a posted NZB before its own articles completed.
-      const existing = await db
-        .from('files')
-        .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? ''])
-        .first()
+      const existing = await db.from('files').where('first_message_id', segments[0]?.id ?? '').first()
       if (existing) {
         await db.from('binaries').where('id', binaryId).update({ file_id: existing.id })
         await dropCopiedParts(binary)
@@ -59,26 +57,23 @@ export default class FileWorker extends BaseCommand {
       }
 
       // file_key is unique: a second completion of the same upload inserts nothing.
-      const knex = db.connection().getWriteClient()
       const now = new Date()
-      const [inserted] = await knex('files')
-        .insert({
+      const [inserted] = await FileStore.insert([
+        {
           file_key: binary.file_key,
           filename: binary.name,
           poster: binary.poster,
           date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(),
           parts: parts.length,
-          message_ids: JSON.stringify(messageIds),
+          segments,
           groups: JSON.stringify(binary.groups),
           release_id: binary.release_id,
           real_name: binary.real_name,
           size: parts.reduce((sum, part) => sum + Number(part.bytes), 0),
           created_at: now,
           updated_at: now,
-        })
-        .onConflict('file_key')
-        .ignore()
-        .returning('id')
+        },
+      ])
 
       const fileId =
         inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id
@@ -91,8 +86,21 @@ export default class FileWorker extends BaseCommand {
       }
 
       this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`)
+      // An SFV posted earlier may already list it (D34).
+      await SfvService.claim({
+        id: Number(fileId),
+        name: binary.real_name ?? binary.name,
+        poster: binary.poster,
+        date: binary.posted_at ? new Date(binary.posted_at).getTime() : null,
+        release_id: binary.release_id,
+      })
       await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` })
-    }, { connection: queueConfig.connection })
+    }, {
+      connection: queueConfig.connection,
+      // One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and
+      // the file insert ignores duplicates, so they can run side by side.
+      concurrency: env.get('FILE_CONCURRENCY', 8),
+    })
 
     worker.on('failed', (job, err) => {
       this.logger.error(`File job ${job?.id} failed: ${err.message}`)
@@ -112,6 +120,13 @@ export default class FileWorker extends BaseCommand {
  * ones keep them, so articles of an equal-sized next volume are recognised
  * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17).
  */
-async function dropCopiedParts(binary: { id: number; size: number | null }) {
-  if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete()
+/**
+ * The file holds the segment list now. Header-derived parts (no per-part
+ * CRC) aren't needed any more; body-derived ones are kept for split-volume
+ * verification (D20). Going by the CRC, not the binary's size: content
+ * inspection gives header-derived binaries a size too, which kept ~20M
+ * parts around (LOG.md 2026-10-02).
+ */
+async function dropCopiedParts(binary: { id: number }) {
+  await db.from('parts').where('binary_id', binary.id).whereNull('pcrc32').delete()
 }

commands/workers/Spots.ts

@@ -6,6 +6,10 @@ import NntpService from '#services/NntpService'
 import SpotService from '#services/SpotService'
 
 const IDLE_MS = 30_000
+/** Spots claimed at a time; most of a history backfill is listed in bulk. */
+const BATCH = 500
+/** Imported spots whose details are read per idle round. */
+const DETAILS_BATCH = 50
 
 /**
  * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
@@ -29,17 +33,22 @@ export default class SpotsWorker extends BaseCommand {
     let done = 0
     try {
       while (!Shutdown.requested && (!this.limit || done < this.limit)) {
-        const batch = await SpotService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
+        const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
         if (batch.length === 0) {
           if (this.limit) break
-          await Shutdown.sleep(IDLE_MS)
+          // Idle: read the details of imported spots that don't have them yet (newest first).
+          if ((await SpotService.fillDetails(DETAILS_BATCH)) === 0) await Shutdown.sleep(IDLE_MS)
           continue
         }
+        // Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
+        const listable = batch.filter((spot) => SpotService.isListable(spot))
+        const single = batch.filter((spot) => !SpotService.isListable(spot))
+        counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
         let next = 0
         await Promise.all(
           Array.from({ length: concurrency }, async () => {
-            while (next < batch.length) {
-              const status = await SpotService.process(batch[next++])
+            while (next < single.length) {
+              const status = await SpotService.process(single[next++])
               counts[status] = (counts[status] ?? 0) + 1
             }
           })