usenet-indexer/app/services/planner.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import coordinatorConfig from '#config/coordinator'
import NntpPool from '#services/nntp_pool'

/**
 * The indexed groups and their ranges (REBUILD.md, group list, coordinator).
 * New groups get priorities after the existing ones (owner rule). A group's
 * span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is
 * its position in its group's history (0 oldest … 1000 newest), so all groups
 * start with their oldest era. Planning again only adds ranges for new articles.
 */
class Planner {
  async addGroups(names: string[]): Promise<number> {
    let added = 0
    for (const name of names) {
      const { rows } = await db.rawQuery(
        `INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`,
        [name]
      )
      added += rows.length
    }
    return added
  }

  /** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */
  async largestGroups(pattern: string, top: number): Promise<string[]> {
    const groups = await NntpPool.use((client) => client.listActive(pattern))
    return groups
      .filter((g) => g.high > g.low)
      .sort((a, b) => b.high - b.low - (a.high - a.low))
      .slice(0, top)
      .map((g) => g.name)
  }

  async planGroup(group: { id: number; name: string }): Promise<number> {
    const info = await NntpPool.use((client) => client.group(group.name))
    const size = coordinatorConfig.rangeArticles
    const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
    const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
    const span = Math.max(1, info.high - info.low)
    const rows = []
    for (let start = from; start <= info.high; start += size) {
      rows.push({
        group_id: group.id,
        start_id: start,
        end_id: Math.min(start + size - 1, info.high),
        next_id: start,
        priority: Math.floor((1000 * (start - info.low)) / span),
      })
    }
    for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
    await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
    return rows.length
  }

  /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
  async planAll(): Promise<{ groups: number; ranges: number }> {
    const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
    let ranges = 0
    for (const group of groups) {
      // One group failing (e.g. gone from the server) doesn't stop the others.
      try {
        ranges += await this.planGroup({ id: Number(group.id), name: group.name })
      } catch (error: any) {
        logger.warn(`Planning ${group.name}: ${error.message}`)
      }
    }
    return { groups: groups.length, ranges }
  }

  /**
   * The live tail: ranges for the articles after each planned group's last
   * range, at the tail's priority (before any history). The newest article
   * numbers come from LIST ACTIVE, one command per name prefix (GROUP per
   * group took minutes for 1000 groups).
   */
  async planTail(now = new Date()): Promise<TailPlanned> {
    const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
    if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
    const { rows: ends } = await db.rawQuery(
      `SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
              (status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
       FROM ranges ORDER BY group_id, start_id DESC`,
      [coordinatorConfig.tail.priority]
    )
    const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
      ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
    )
    const highs = new Map<string, number>()
    await NntpPool.use(async (client) => {
      for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
        for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
      }
    })
    const plan = tailRanges(
      groups.map((g) => {
        const l = last.get(Number(g.id))
        return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
      }),
      now,
      { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
    )
    return this.applyTail(plan, now)
  }

  /** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
  async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
    const rows = [...plan.rows]
    let extended = 0
    if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
    await db.transaction(async (trx) => {
      for (const e of plan.extended) {
        const { rows: done } = await trx.rawQuery(
          `UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
          [e.end_id, e.id, e.from - 1]
        )
        if (done.length > 0) extended++
        else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
      }
      if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
      const highs = new Map<number, number>()
      for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
      for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
    })
    return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
  }

  /**
   * Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
   * RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
   * (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
   */
  async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
    const { rows: candidates } = await db.rawQuery(
      `SELECT id, group_id, start_id, end_id, priority FROM ranges
       WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
       ORDER BY group_id, start_id`
    )
    const merges = tailMerges(
      candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
      coordinatorConfig.rangeArticles
    )
    const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
    if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
    await db.transaction(async (trx) => {
      // The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
      await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
      const { rows: gone } = await trx.rawQuery(
        `DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
        [merges.flatMap((m) => m.drop)]
      )
      if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
      const { rows: grown } = await trx.rawQuery(
        `UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
         WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
        [merges.map((m) => m.id), merges.map((m) => m.end_id)]
      )
      if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
    })
    return { candidates: candidates.length, merged: merges.length, dropped }
  }
}

export interface TailPlanned {
  groups: number
  /** Rows added. */
  ranges: number
  /** Open ranges grown. */
  extended: number
}

/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
export function wildmatOf(name: string): string {
  const parts = name.split('.')
  return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
}

export interface TailGroup {
  id: number
  /** End of its last range (null: none, then it has no tail yet). */
  plannedTo: number | null
  /** When its last range was planned. */
  plannedAt: Date
  /** The server's newest article (null: not listed). */
  high: number | null
  /**
   * Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
   * nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
   */
  open?: { id: number; start: number } | null
}

export interface TailRow {
  group_id: number
  start_id: number
  end_id: number
  next_id: number
  priority: number
}

export interface TailPlan {
  /** New ranges. */
  rows: TailRow[]
  /** Open ranges grown to end_id (from: the first new article, the old end + 1). */
  extended: { id: number; group_id: number; from: number; end_id: number }[]
}

/**
 * The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
 * the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
 * tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
 */
export function tailRanges(
  groups: TailGroup[],
  now: Date,
  options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
): TailPlan {
  const rows: TailRow[] = []
  const extended: TailPlan['extended'] = []
  for (const g of groups) {
    if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
    const waiting = g.high - g.plannedTo
    const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
    if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
    let start = g.plannedTo + 1
    if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
      const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
      extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
      start = end + 1
    }
    for (; start <= g.high; start += options.rangeArticles) {
      rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
    }
  }
  return { rows, extended }
}

export interface TailMerge {
  /** The run's first range, grown to end_id. */
  id: number
  end_id: number
  /** The ranges it absorbed. */
  drop: number[]
}

/**
 * Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
 * `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
 */
export function tailMerges(
  ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
  rangeArticles: number
): TailMerge[] {
  const merges: TailMerge[] = []
  let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
  const flush = () => {
    if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
  }
  for (const r of ranges) {
    if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
      cur.end_id = r.end_id
      cur.drop.push(r.id)
      continue
    }
    flush()
    cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
  }
  flush()
  return merges
}

export default new Planner()