usenet-indexer/commands/GcScheduler.ts

import { BaseCommand, flags } from '@adonisjs/core/ace'
import { CommandOptions } from '@adonisjs/core/types/ace'
import type { Queue } from 'bullmq'
import db from '@adonisjs/lucid/services/db'
import gcConfig from '#config/gc'
import QueueService from '#services/QueueService'
import RedisService from '#services/RedisService'
import IndexMaintenance from '#services/IndexMaintenance'
import ReleaseMergeService from '#services/ReleaseMergeService'
import StagingService from '#services/StagingService'
import Shutdown from '#services/Shutdown'

const CLEAN_BATCH = 1000
const SCAN_COUNT = 1000

export default class GcScheduler extends BaseCommand {
  public static commandName = 'gc:scheduler'
  public static description = 'Periodically removes old BullMQ jobs and stale binaries.'

  public static options: CommandOptions = {
    startApp: true,
  }

  @flags.boolean({ description: 'Run one sweep and exit' })
  declare once: boolean

  @flags.boolean({ description: 'Report what would be removed without changing anything' })
  declare dryRun: boolean

  public async run() {
    this.logger.info(
      `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
    )

    try {
      do {
        try {
          await this.sweep()
        } catch (err: any) {
          this.logger.error(`GC sweep failed: ${err.message}`)
        }
        if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
      } while (!this.once && !Shutdown.requested)
    } finally {
      await QueueService.closeAll()
      await RedisService.close()
    }
  }

  private async sweep() {
    const { completedAgeSeconds, failedAgeSeconds } = gcConfig.jobs
    for (const queue of [...QueueService.all, ...QueueService.legacy]) {
      const completed = await this.cleanJobs(queue, 'completed', completedAgeSeconds)
      const failed = await this.cleanJobs(queue, 'failed', failedAgeSeconds)
      if (completed || failed) {
        this.logger.info(
          `${queue.name}: ${this.verb('remove', 'removed')} ${completed} completed and ${failed} failed jobs.`
        )
      }
    }

    const trimmed = await this.trimBodyQueue()
    if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)

    const binaries = await this.deleteStaleBinaries()
    this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)

    if (!this.dryRun) {
      // Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
      const budget = () => Date.now() + gcConfig.deleteBudgetMs
      const parts = await this.dropCopiedParts(budget())
      const single = await this.deleteSingleArticleBinaries(budget())
      const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
      const releases = await this.deleteEmptyReleases(budget())
      const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
      if (staging.adopted || staging.expired) {
        this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
      }
      if (parts || single || unfingerprinted || releases) {
        this.logger.info(
          `Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
            `${releases} empty releases deleted.`
        )
      }
    }

    const deferred = await this.deleteOldDeferredFiles()
    if (deferred) this.logger.info(`Deferred files: ${this.verb('delete', 'deleted')} ${deferred} past retention.`)

    const { expired, scanned } = await this.expireLegacyPendingFiles()
    if (scanned > 0) {
      this.logger.info(
        `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
      )
    }

    // Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep.
    const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
    if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)

    await this.rebuildIndexes()
  }

  /** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
  private async rebuildIndexes() {
    const key = 'gc:reindex:last'
    const last = Number((await RedisService.client.get(key)) ?? 0)
    if (Date.now() - last < gcConfig.reindexIntervalMs) return
    const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
    if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
    const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
    this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
  }

  /**
   * Drops the oldest waiting body jobs above gcConfig.bodyQueueMax. BullMQ
   * workers take jobs from the right end of the wait list, so the oldest are
   * there.
   */
  private async trimBodyQueue(): Promise<number> {
    const redis = RedisService.client
    const key = QueueService.bodyQueue.toKey('wait')
    const excess = (await redis.llen(key)) - gcConfig.bodyQueueMax
    if (excess <= 0 || this.dryRun) return Math.max(excess, 0)
    let dropped = 0
    while (dropped < excess) {
      const ids: string[] = (await redis.rpop(key, Math.min(5000, excess - dropped))) ?? []
      if (ids.length === 0) break
      await redis.unlink(...ids.map((id) => QueueService.bodyQueue.toKey(id)))
      dropped += ids.length
    }
    return dropped
  }

  /**
   * Deletes binaries (and, by cascade, their parts) that received no segment
   * for `staleAfterSeconds`: incomplete ones are given up on; completed ones
   * already live on in `files`; conflicted ones will never become a file.
   * Completed ones still waiting for worker:file are kept.
   */
  private async deleteStaleBinaries(): Promise<number> {
    const query = db
      .from('binaries')
      .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000))
      .where((q) =>
        q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at')
      )

    if (this.dryRun) {
      const [{ count }] = await query.count('* as count')
      return Number(count)
    }
    return Number(await query.delete())
  }

  private async deleteOldDeferredFiles(): Promise<number> {
    const cutoff = new Date(Date.now() - gcConfig.deferredRetentionDays * 24 * 60 * 60 * 1000)
    let total = 0
    for (const [table, column] of [['deferred_files', 'first_seen_at'], ['deferred_ranges', 'created_at']]) {
      const query = db.from(table).where(column, '<', cutoff)
      if (this.dryRun) {
        const [{ count }] = await query.count('* as count')
        total += Number(count)
      } else {
        total += Number(await query.delete())
      }
    }
    return total
  }

  // Keyset cursors over the tables, kept between sweeps so one pass over
  // millions of rows is spread over several sweeps.
  private singleCursor = 0
  private releaseCursor = 0

  /** Parts of header-derived binaries that are a file already (worker:file now drops them itself). */
  private async dropCopiedParts(budget: number): Promise<number> {
    let total = 0
    while (Date.now() < budget) {
      const result = await db.rawQuery(
        // Header-derived parts (no per-part CRC) of binaries that are a file now.
        `DELETE FROM parts WHERE pcrc32 IS NULL AND binary_id IN (
           SELECT b.id FROM binaries b
           WHERE b.file_id IS NOT NULL
             AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id AND p.pcrc32 IS NULL)
           LIMIT ?)`,
        [500]
      )
      total += result.rowCount ?? 0
      if (!result.rowCount) break
    }
    return total
  }

  /** Header-derived binaries that never got a second article (gcConfig.binaries.singleArticleAfterSeconds). */
  private async deleteSingleArticleBinaries(budget: number): Promise<number> {
    const cutoff = new Date(Date.now() - gcConfig.binaries.singleArticleAfterSeconds * 1000)
    let total = 0
    while (Date.now() < budget) {
      const { rows } = await db.rawQuery(
        `WITH batch AS (
           SELECT id FROM binaries WHERE id > ? ORDER BY id LIMIT ?
         ), gone AS (
           DELETE FROM binaries b USING batch
           WHERE b.id = batch.id AND b.completed_at IS NULL AND b.size IS NULL AND b.total_parts > 1
             AND b.last_seen_at < ? AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) <= 1
           RETURNING 1)
         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
        [this.singleCursor, gcConfig.deleteBatch, cutoff]
      )
      total += Number(rows[0].deleted)
      if (rows[0].last === null) {
        this.singleCursor = 0
        break
      }
      this.singleCursor = Number(rows[0].last)
    }
    return total
  }

  /**
   * Completed header-derived binaries without a fingerprint, some hours after
   * completion: their file exists, their parts are gone, and par2 naming needs
   * a size and MD5 they don't have. Oldest first, via the last_seen_at index.
   */
  private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
    const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
    let total = 0
    while (Date.now() < budget) {
      const result = await db.rawQuery(
        `DELETE FROM binaries WHERE id IN (
           SELECT id FROM binaries
           WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
             AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
           ORDER BY last_seen_at LIMIT ?)`,
        [cutoff, gcConfig.deleteBatch]
      )
      total += result.rowCount
      if (result.rowCount < gcConfig.deleteBatch) break
    }
    return total
  }

  /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
  private async deleteEmptyReleases(budget: number): Promise<number> {
    const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)
    let total = 0
    while (Date.now() < budget) {
      const { rows } = await db.rawQuery(
        `WITH batch AS (
           SELECT id FROM releases WHERE id > ? ORDER BY id LIMIT ?
         ), gone AS (
           DELETE FROM releases r USING batch
           WHERE r.id = batch.id AND r.file_count = 0 AND r.first_seen_at < ?
             AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
             AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)
             AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)
             AND NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)
           RETURNING 1)
         SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
        [this.releaseCursor, gcConfig.deleteBatch, cutoff]
      )
      total += Number(rows[0].deleted)
      if (rows[0].last === null) {
        this.releaseCursor = 0
        break
      }
      this.releaseCursor = Number(rows[0].last)
    }
    return total
  }

  /** Removes jobs in `state` that finished more than `ageSeconds` ago. */
  private async cleanJobs(queue: Queue, state: 'completed' | 'failed', ageSeconds: number) {
    const graceMs = ageSeconds * 1000
    if (this.dryRun) {
      const cutoff = Date.now() - graceMs
      // Finished-job sets are sorted by finishedOn, so a score range counts them cheaply.
      return RedisService.client.zcount(queue.toKey(state), '-inf', cutoff)
    }

    let total = 0
    let removed: string[]
    do {
      removed = await queue.clean(graceMs, CLEAN_BATCH, state)
      total += removed.length
    } while (removed.length === CLEAN_BATCH)
    return total
  }

  /**
   * Pending-file hashes of the old header worker (and the pre-AdonisJS app)
   * never expire. Give them a TTL rather than deleting them outright.
   */
  private async expireLegacyPendingFiles() {
    const redis = RedisService.client
    const { prefix, ttlSeconds } = gcConfig.legacyPendingFiles
    let cursor = '0'
    let scanned = 0
    let expired = 0

    do {
      const [next, keys]: [string, string[]] = await redis.scan(
        cursor,
        'MATCH',
        `${prefix}*`,
        'COUNT',
        SCAN_COUNT
      )
      cursor = next
      if (keys.length === 0) continue
      scanned += keys.length

      const ttls: [Error | null, number][] = await keys
        .reduce((pipeline, key) => pipeline.ttl(key), redis.pipeline())
        .exec()
      const orphaned = keys.filter((_, i) => ttls[i][1] === -1)
      expired += orphaned.length

      if (!this.dryRun && orphaned.length > 0) {
        await orphaned
          .reduce((pipeline, key) => pipeline.expire(key, ttlSeconds, 'NX'), redis.pipeline())
          .exec()
      }
    } while (cursor !== '0')

    return { scanned, expired }
  }

  private verb(present: string, past: string) {
    return this.dryRun ? `would ${present}` : past
  }
}