usenet-indexer/app/services/prefetcher.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import coordinatorConfig from '#config/coordinator'
import NntpPool from '#services/nntp_pool'
import Spool, { type ChunkSpan } from '#services/spool'

interface Target {
  id: number
  group: string
  /** Chunk starts to fetch without compression (ranges.plain_chunks). */
  plain: Set<number>
  /** Chunks not spooled yet, in order. */
  missing: ChunkSpan[]
  /** All of the range's chunks, from its start. */
  total: number
}

/** How often the targets are read again from the database. */
const REFRESH_MS = 2000
/** How often the spool's size is measured (a stat per file). */
const SIZE_MS = 30_000

/**
 * Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
 * Every connection takes the next missing chunk: the leased ranges' first
 * (a worker waits for them; all connections work on the same range, in
 * order), then the next planned ranges, about SPOOL_AHEAD_PER_LEASE per
 * active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
 * connection fills ~26 k overview lines/s, so a range on one connection
 * kept its worker waiting (LOG.md 2026-10-09).
 */
export class Prefetcher {
  /**
   * This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
   * process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
   * LOG.md 2026-10-09), less than 50 connections bring.
   */
  constructor(private shard = { index: 0, count: 1 }) {}

  private targets: Target[] = []
  private claimed = new Set<string>()
  private running = 0
  private refreshedAt = 0
  private stopped = false
  private spoolBytes = 0
  private sizedAt = 0
  /** Fetched since the last report: chunks, articles, ms spent fetching. */
  private stats = { chunks: 0, articles: 0, fetchMs: 0, since: Date.now() }

  /** One round: refreshes the targets now and then, and puts idle connections to work. */
  async tick() {
    if (Date.now() - this.refreshedAt >= REFRESH_MS) {
      this.refreshedAt = Date.now()
      this.targets = await this.readTargets()
    }
    this.report()
    while (!this.stopped && this.running < NntpPool.size && this.peek()) {
      this.running++
      this.work()
        .catch((error) => logger.warn(`Prefetching: ${error.message}`))
        .finally(() => this.running--)
    }
  }

  stop() {
    this.stopped = true
  }

  get active() {
    return this.running
  }

  /** One connection's loop: chunks until there is nothing left to fetch. */
  private async work() {
    while (!this.stopped) {
      const next = this.take()
      if (!next) return
      const { target, span } = next
      try {
        const fetchedAt = Date.now()
        const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
          await client.group(target.group)
          return client.xoverRaw(span.from, span.to)
        }
        const data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await NntpPool.use(fetch)
        this.stats.chunks++
        this.stats.articles += span.to - span.from + 1
        this.stats.fetchMs += Date.now() - fetchedAt
        const existed = await Spool.has(target.id, span)
        await Spool.write(target.id, span, data)
        if (existed) continue
        this.spoolBytes += data.length
        const { rows } = await db.rawQuery(
          `UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
          [target.id]
        )
        if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
      } finally {
        this.claimed.delete(`${target.id}:${span.from}`)
      }
    }
  }

  /** A line a minute: what the connections fetched, and how long a chunk took. */
  private report() {
    const seconds = (Date.now() - this.stats.since) / 1000
    if (seconds < 60) return
    const { chunks, articles, fetchMs } = this.stats
    if (chunks > 0) {
      logger.info(
        `Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
          `${(fetchMs / chunks / 1000).toFixed(1)} s a chunk, connections busy ${Math.round((fetchMs / (seconds * 1000 * NntpPool.size)) * 100)}%.`
      )
    }
    this.stats = { chunks: 0, articles: 0, fetchMs: 0, since: Date.now() }
  }

  private peek(): boolean {
    return this.targets.some((t) => t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
  }

  /** The first unclaimed missing chunk, in target order. */
  private take(): { target: Target; span: ChunkSpan } | null {
    for (const target of this.targets) {
      const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
      if (index === -1) continue
      const [span] = target.missing.splice(index, 1)
      this.claimed.add(`${target.id}:${span.from}`)
      return { target, span }
    }
    return null
  }

  private async readTargets(): Promise<Target[]> {
    if (Date.now() - this.sizedAt >= SIZE_MS) {
      this.sizedAt = Date.now()
      this.spoolBytes = await Spool.size()
    }
    const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
    const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks'))
    // Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
    const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
    // Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
    const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
    const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
    const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
    const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
    const fresh = room && Number(ready) < ahead
      ? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(ahead - Number(ready))
      : []
    const targets: Target[] = []
    for (const row of [...leased, ...(room ? started : []), ...fresh]) {
      const start = Number(row.start_id)
      const end = Number(row.end_id)
      const total = Spool.chunksOf({ start, end }).length
      const missing: ChunkSpan[] = []
      for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
        if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
      }
      if (missing.length > 0) targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total })
    }
    return targets
  }
}