import { BaseCommand } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import Group from '#models/group' import NntpService from '#services/NntpService' import QueueService from '#services/QueueService' export default class IndexScheduler extends BaseCommand { public static commandName = 'index:scheduler' public static description = 'Periodically checks for new articles and schedules them for fetching.' public static options: CommandOptions = { startApp: true, } private pool = NntpService private fetchQueue = QueueService.nntpFetchQueue public async run() { this.logger.info('Scheduler started. Awaiting tasks...') const schedule = async () => { this.logger.info('Checking for new headers...') const groups = await Group.query().where('active', true) if (groups.length === 0) { this.logger.info('No active groups to index. Add some via `node ace db:seed` or manually.') return } let conn try { conn = await this.pool.acquire() for (const group of groups) { try { const groupInfo: any = await conn.group(group.name) // nntp-js returns article numbers as strings. We must parse them to BigInts. const firstArticle = BigInt(groupInfo.first) const lastArticle = BigInt(groupInfo.last) // lastIndexedId from the database should also be treated as a BigInt. const lastIndexed = group.lastIndexedId ? BigInt(group.lastIndexedId) : null const startId = lastIndexed ? lastIndexed + 1n : firstArticle if (startId > lastArticle) { this.logger.info(`No new headers for group ${group.name}.`) continue } const BATCH_SIZE = 100000n const proposedEndId = startId + BATCH_SIZE - 1n const endId = proposedEndId < lastArticle ? proposedEndId : lastArticle this.logger.info(`Queueing fetch job for ${group.name}: articles ${startId} to ${endId}`) await this.fetchQueue.add('fetch-headers', { groupName: group.name, startId: startId.toString(), endId: endId.toString(), }) group.lastIndexedId = endId await group.save() } catch (err: any) { this.logger.error(`Error processing group ${group.name}: ${err.message}`) } } } catch (err: any) { this.logger.error(`Error in scheduler main loop: ${err.message}`) } finally { if (conn) { this.pool.release(conn) } } } // Run once immediately and then on an interval. schedule() setInterval(schedule, 60000) // 1 minute // Keep the command running await new Promise(() => {}) } }