usenet-indexer/app/services/BackfillService.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import indexerConfig from '#config/indexer'
import NntpService from '#services/NntpService'
import FetchRangeService from '#services/FetchRangeService'
import { articleAtDate } from '#lib/nntp/date_search'

/** Articles XOVERed to find a posting date near an article number. */
const PROBE = 200n

/**
 * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
 * where live indexing began back to the article posted N days ago; `tick`
 * (every scheduler run) queues low-priority, header-only fetch ranges from
 * that cursor downwards, newest first, while few are waiting and the
 * database stays within its size limits.
 */
class BackfillService {
  /** Sets the group's backfill to reach back `days` days. Returns its range. */
  async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
    const group = await db.from('groups').where('name', groupName).first()
    if (!group) throw new Error(`Unknown group ${groupName}`)
    const conn = await NntpService.acquire()
    let failed = false
    try {
      const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
      const dateOf = async (n: bigint) => {
        const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
        const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
        const date = header ? Date.parse(header.date) : Number.NaN
        return Number.isNaN(date) ? null : date
      }
      const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
      // Live indexing began a little before the group was added: continue from there.
      const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
      const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
      await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
      return next > until ? { from: until, to: next } : null
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
  }

  /** Stops a group's backfill (queued ranges still run). */
  async stop(groupName: string) {
    await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
  }

  /** Queues the next backfill ranges. Returns how many. */
  async tick(): Promise<number> {
    const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
    const groups = await db
      .from('groups')
      .where('active', true)
      .whereNotNull('backfill_until_id')
      .whereRaw('backfill_next_id > backfill_until_id')
      .orderBy('id')
    if (groups.length === 0) return 0

    const room = await this.room(maxDefaultGb, maxHddGb)
    if (room !== true) {
      logger.warn(`Backfill paused: ${room}.`)
      return 0
    }
    const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
    let free = maxQueued - Number(count)
    let queued = 0
    // Round-robin over the groups, one range each per pass.
    while (free > 0 && groups.length > 0) {
      for (const group of [...groups]) {
        if (free <= 0) break
        const next = BigInt(group.backfill_next_id)
        const until = BigInt(group.backfill_until_id)
        const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
        await FetchRangeService.queue(group, start, next, { backfill: true })
        group.backfill_next_id = (start - 1n).toString()
        await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
        free--
        queued++
        if (start - 1n <= until) {
          groups.splice(groups.indexOf(group), 1)
          logger.info(`${group.name}: backfill fully queued.`)
        }
      }
    }
    return queued
  }

  /** True when both parts of the database are under their limits, else why not. */
  /**
   * Whether the database has room for more history: the default tablespace
   * (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed
   * leases (CONCEPT.md D38) check the same limits.
   */
  async room(
    maxDefaultGb = indexerConfig.backfill.maxDefaultGb,
    maxHddGb = indexerConfig.backfill.maxHddGb
  ): Promise<true | string> {
    const { rows } = await db.rawQuery(
      `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
    )
    const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
    if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
    if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
    return true
  }
}

export default new BackfillService()