usenet-indexer/app/services/IndexMaintenance.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'

/** Only indexes at least this big are worth checking. */
const MIN_BYTES = 50 * 1024 * 1024
/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
const MAX_LEAF_DENSITY = 50

export interface Rebuilt {
  index: string
  density: number
  before: number
  after: number
}

/**
 * GC deletes millions of rows (single-article binaries, copied parts, empty
 * releases). Vacuum makes their index entries reusable but doesn't compact
 * the index, so B-trees on the churning tables end up 10–20% full (LOG.md
 * 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
 * D33). Needs the pgstattuple extension.
 */
class IndexMaintenance {
  async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
    const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
    if (extension.length === 0) {
      logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
      return []
    }
    if (!dryRun) await this.dropLeftovers()

    const { rows: candidates } = await db.rawQuery(
      `SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
       FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
       JOIN pg_namespace n ON n.oid = c.relnamespace
       WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
       ORDER BY 2 DESC`,
      [MIN_BYTES]
    )
    const rebuilt: Rebuilt[] = []
    for (const { index, bytes } of candidates) {
      const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
      const density = Number(rows[0].avg_leaf_density)
      if (density >= MAX_LEAF_DENSITY) continue
      if (!dryRun && !(await this.reindex(index))) continue
      const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
      const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
      rebuilt.push(result)
      logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
    }
    return rebuilt
  }

  /**
   * REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
   * binaries during a backfill). Its half-built copy is dropped and it is
   * tried once more; a second failure skips the index until the next run.
   */
  /** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
  private async dropLeftovers(index = '') {
    const { rows } = await db.rawQuery(
      `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
       WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
      [index]
    )
    for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
  }

  private async reindex(index: string): Promise<boolean> {
    for (let attempt = 1; attempt <= 2; attempt++) {
      try {
        await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
        return true
      } catch (error: any) {
        logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
        await this.dropLeftovers(index)
      }
    }
    return false
  }
}

const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`

export default new IndexMaintenance()