import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import coordinatorConfig from '#config/coordinator' import NntpPool from '#services/nntp_pool' /** * The indexed groups and their ranges (REBUILD.md, group list, coordinator). * New groups get priorities after the existing ones (owner rule). A group's * span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is * its position in its group's history (0 oldest … 1000 newest), so all groups * start with their oldest era. Planning again only adds ranges for new articles. */ class Planner { async addGroups(names: string[]): Promise { let added = 0 for (const name of names) { const { rows } = await db.rawQuery( `INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`, [name] ) added += rows.length } return added } /** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */ async largestGroups(pattern: string, top: number): Promise { const groups = await NntpPool.use((client) => client.listActive(pattern)) return groups .filter((g) => g.high > g.low) .sort((a, b) => b.high - b.low - (a.high - a.low)) .slice(0, top) .map((g) => g.name) } async planGroup(group: { id: number; name: string }): Promise { const info = await NntpPool.use((client) => client.group(group.name)) const size = coordinatorConfig.rangeArticles const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first() const from = Math.max(info.low, Number(last?.end ?? 0) + 1) const span = Math.max(1, info.high - info.low) const rows = [] for (let start = from; start <= info.high; start += size) { rows.push({ group_id: group.id, start_id: start, end_id: Math.min(start + size - 1, info.high), next_id: start, priority: Math.floor((1000 * (start - info.low)) / span), }) } for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000)) await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() }) return rows.length } /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */ async planAll(): Promise<{ groups: number; ranges: number }> { const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name') let ranges = 0 for (const group of groups) { // One group failing (e.g. gone from the server) doesn't stop the others. try { ranges += await this.planGroup({ id: Number(group.id), name: group.name }) } catch (error: any) { logger.warn(`Planning ${group.name}: ${error.message}`) } } return { groups: groups.length, ranges } } /** * The live tail: ranges for the articles after each planned group's last * range, at the tail's priority (before any history). The newest article * numbers come from LIST ACTIVE, one command per name prefix (GROUP per * group took minutes for 1000 groups). */ async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> { const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at') if (groups.length === 0) return { groups: 0, ranges: 0 } const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`) const plannedTo = new Map(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)])) const highs = new Map() await NntpPool.use(async (client) => { for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) { for (const g of await client.listActive(pattern)) highs.set(g.name, g.high) } }) const rows = tailRanges( groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })), now, { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail } ) if (rows.length === 0) return { groups: 0, ranges: 0 } await db.transaction(async (trx) => { await trx.table('ranges').multiInsert(rows) for (const id of new Set(rows.map((r) => r.group_id))) { const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id)) await trx.from('groups').where('id', id).update({ high, planned_at: now }) } }) return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length } } } /** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */ export function wildmatOf(name: string): string { const parts = name.split('.') return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name } export interface TailGroup { id: number /** End of its last range (null: none, then it has no tail yet). */ plannedTo: number | null /** When its last range was planned. */ plannedAt: Date /** The server's newest article (null: not listed). */ high: number | null } /** * The tail's new ranges: per group, the articles after its last range, in * ranges of at most `rangeArticles`, once there are `minArticles` of them or * the group's last planning is `maxDelayMinutes` ago. */ export function tailRanges( groups: TailGroup[], now: Date, options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number } ) { const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = [] for (const g of groups) { if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue const waiting = g.high - g.plannedTo const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000 if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) { rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority }) } } return rows } export default new Planner()