usenet-indexer/app/services/planner.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import coordinatorConfig from '#config/coordinator'
import NntpPool from '#services/nntp_pool'

/**
 * The indexed groups and their ranges (REBUILD.md, group list, coordinator).
 * New groups get priorities after the existing ones (owner rule). A group's
 * span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is
 * its position in its group's history (0 oldest … 1000 newest), so all groups
 * start with their oldest era. Planning again only adds ranges for new articles.
 */
class Planner {
  async addGroups(names: string[]): Promise<number> {
    let added = 0
    for (const name of names) {
      const { rows } = await db.rawQuery(
        `INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`,
        [name]
      )
      added += rows.length
    }
    return added
  }

  /** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */
  async largestGroups(pattern: string, top: number): Promise<string[]> {
    const groups = await NntpPool.use((client) => client.listActive(pattern))
    return groups
      .filter((g) => g.high > g.low)
      .sort((a, b) => b.high - b.low - (a.high - a.low))
      .slice(0, top)
      .map((g) => g.name)
  }

  async planGroup(group: { id: number; name: string }): Promise<number> {
    const info = await NntpPool.use((client) => client.group(group.name))
    const size = coordinatorConfig.rangeArticles
    const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
    const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
    const span = Math.max(1, info.high - info.low)
    const rows = []
    for (let start = from; start <= info.high; start += size) {
      rows.push({
        group_id: group.id,
        start_id: start,
        end_id: Math.min(start + size - 1, info.high),
        next_id: start,
        priority: Math.floor((1000 * (start - info.low)) / span),
      })
    }
    for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
    await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
    return rows.length
  }

  /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
  async planAll(): Promise<{ groups: number; ranges: number }> {
    const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
    let ranges = 0
    for (const group of groups) {
      // One group failing (e.g. gone from the server) doesn't stop the others.
      try {
        ranges += await this.planGroup({ id: Number(group.id), name: group.name })
      } catch (error: any) {
        logger.warn(`Planning ${group.name}: ${error.message}`)
      }
    }
    return { groups: groups.length, ranges }
  }

  /**
   * The live tail: ranges for the articles after each planned group's last
   * range, at the tail's priority (before any history). The newest article
   * numbers come from LIST ACTIVE, one command per name prefix (GROUP per
   * group took minutes for 1000 groups).
   */
  async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
    if (groups.length === 0) return { groups: 0, ranges: 0 }
    const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
    const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
    const highs = new Map<string, number>()
    await NntpPool.use(async (client) => {
      for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
        for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
      }
    })
    const rows = tailRanges(
      groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
      now,
      { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
    )
    if (rows.length === 0) return { groups: 0, ranges: 0 }
    await db.transaction(async (trx) => {
      await trx.table('ranges').multiInsert(rows)
      for (const id of new Set(rows.map((r) => r.group_id))) {
        const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
        await trx.from('groups').where('id', id).update({ high, planned_at: now })
      }
    })
    return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
  }
}

/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
export function wildmatOf(name: string): string {
  const parts = name.split('.')
  return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
}

export interface TailGroup {
  id: number
  /** End of its last range (null: none, then it has no tail yet). */
  plannedTo: number | null
  /** When its last range was planned. */
  plannedAt: Date
  /** The server's newest article (null: not listed). */
  high: number | null
}

/**
 * The tail's new ranges: per group, the articles after its last range, in
 * ranges of at most `rangeArticles`, once there are `minArticles` of them or
 * the group's last planning is `maxDelayMinutes` ago.
 */
export function tailRanges(
  groups: TailGroup[],
  now: Date,
  options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
) {
  const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
  for (const g of groups) {
    if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
    const waiting = g.high - g.plannedTo
    const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
    if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
    for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
      rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
    }
  }
  return rows
}

export default new Planner()