import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import coordinatorConfig from '#config/coordinator' import NntpPool from '#services/nntp_pool' /** * The indexed groups and their ranges (REBUILD.md, group list, coordinator). * New groups get priorities after the existing ones (owner rule). A group's * span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is * its position in its group's history (0 oldest … 1000 newest), so all groups * start with their oldest era. Planning again only adds ranges for new articles. */ class Planner { async addGroups(names: string[]): Promise { let added = 0 for (const name of names) { const { rows } = await db.rawQuery( `INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`, [name] ) added += rows.length } return added } /** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */ async largestGroups(pattern: string, top: number): Promise { const groups = await NntpPool.use((client) => client.listActive(pattern)) return groups .filter((g) => g.high > g.low) .sort((a, b) => b.high - b.low - (a.high - a.low)) .slice(0, top) .map((g) => g.name) } async planGroup(group: { id: number; name: string }): Promise { const info = await NntpPool.use((client) => client.group(group.name)) const size = coordinatorConfig.rangeArticles const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first() const from = Math.max(info.low, Number(last?.end ?? 0) + 1) const span = Math.max(1, info.high - info.low) const rows = [] for (let start = from; start <= info.high; start += size) { rows.push({ group_id: group.id, start_id: start, end_id: Math.min(start + size - 1, info.high), next_id: start, priority: Math.floor((1000 * (start - info.low)) / span), }) } for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000)) await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() }) return rows.length } /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */ async planAll(): Promise<{ groups: number; ranges: number }> { const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name') let ranges = 0 for (const group of groups) { // One group failing (e.g. gone from the server) doesn't stop the others. try { ranges += await this.planGroup({ id: Number(group.id), name: group.name }) } catch (error: any) { logger.warn(`Planning ${group.name}: ${error.message}`) } } return { groups: groups.length, ranges } } /** * The live tail: ranges for the articles after each planned group's last * range, at the tail's priority (before any history). The newest article * numbers come from LIST ACTIVE, one command per name prefix (GROUP per * group took minutes for 1000 groups). */ async planTail(now = new Date()): Promise { const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at') if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 } const { rows: ends } = await db.rawQuery( `SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id, (status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open FROM ranges ORDER BY group_id, start_id DESC`, [coordinatorConfig.tail.priority] ) const last = new Map( ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }]) ) const highs = new Map() await NntpPool.use(async (client) => { for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) { for (const g of await client.listActive(pattern)) highs.set(g.name, g.high) } }) const plan = tailRanges( groups.map((g) => { const l = last.get(Number(g.id)) return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null } }), now, { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail } ) return this.applyTail(plan, now) } /** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */ async applyTail(plan: TailPlan, now = new Date()): Promise { const rows = [...plan.rows] let extended = 0 if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 } await db.transaction(async (trx) => { for (const e of plan.extended) { const { rows: done } = await trx.rawQuery( `UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`, [e.end_id, e.id, e.from - 1] ) if (done.length > 0) extended++ else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority }) } if (rows.length > 0) await trx.table('ranges').multiInsert(rows) const highs = new Map() for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id)) for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now }) }) return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended } } /** * Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most * RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail * (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go. */ async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> { const { rows: candidates } = await db.rawQuery( `SELECT id, group_id, start_id, end_id, priority FROM ranges WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL ORDER BY group_id, start_id` ) const merges = tailMerges( candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })), coordinatorConfig.rangeArticles ) const dropped = merges.reduce((n, m) => n + m.drop.length, 0) if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped } await db.transaction(async (trx) => { // The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile. await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`) const { rows: gone } = await trx.rawQuery( `DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`, [merges.flatMap((m) => m.drop)] ) if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`) const { rows: grown } = await trx.rawQuery( `UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`, [merges.map((m) => m.id), merges.map((m) => m.end_id)] ) if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`) }) return { candidates: candidates.length, merged: merges.length, dropped } } } export interface TailPlanned { groups: number /** Rows added. */ ranges: number /** Open ranges grown. */ extended: number } /** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */ export function wildmatOf(name: string): string { const parts = name.split('.') return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name } export interface TailGroup { id: number /** End of its last range (null: none, then it has no tail yet). */ plannedTo: number | null /** When its last range was planned. */ plannedAt: Date /** The server's newest article (null: not listed). */ high: number | null /** * Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that * nothing spooled, took or tried (its start: the cut at rangeArticles counts from there). */ open?: { id: number; start: number } | null } export interface TailRow { group_id: number start_id: number end_id: number next_id: number priority: number } export interface TailPlan { /** New ranges. */ rows: TailRow[] /** Open ranges grown to end_id (from: the first new article, the old end + 1). */ extended: { id: number; group_id: number; from: number; end_id: number }[] } /** * The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or * the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a * tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`. */ export function tailRanges( groups: TailGroup[], now: Date, options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number } ): TailPlan { const rows: TailRow[] = [] const extended: TailPlan['extended'] = [] for (const g of groups) { if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue const waiting = g.high - g.plannedTo const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000 if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue let start = g.plannedTo + 1 if (g.open && g.open.start + options.rangeArticles - 1 >= start) { const end = Math.min(g.open.start + options.rangeArticles - 1, g.high) extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end }) start = end + 1 } for (; start <= g.high; start += options.rangeArticles) { rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority }) } } return { rows, extended } } export interface TailMerge { /** The run's first range, grown to end_id. */ id: number end_id: number /** The ranges it absorbed. */ drop: number[] } /** * Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most * `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything. */ export function tailMerges( ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[], rangeArticles: number ): TailMerge[] { const merges: TailMerge[] = [] let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null const flush = () => { if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop }) } for (const r of ranges) { if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) { cur.end_id = r.end_id cur.drop.push(r.id) continue } flush() cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] } } flush() return merges } export default new Planner()