86 lines
2.8 KiB
TypeScript
86 lines
2.8 KiB
TypeScript
import { BaseCommand } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import Group from '#models/group'
|
|
import NntpService from '#services/NntpService'
|
|
import QueueService from '#services/QueueService'
|
|
|
|
export default class IndexScheduler extends BaseCommand {
|
|
public static commandName = 'index:scheduler'
|
|
public static description = 'Periodically checks for new articles and schedules them for fetching.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
private pool = NntpService
|
|
private fetchQueue = QueueService.nntpFetchQueue
|
|
|
|
public async run() {
|
|
this.logger.info('Scheduler started. Awaiting tasks...')
|
|
|
|
const schedule = async () => {
|
|
this.logger.info('Checking for new headers...')
|
|
const groups = await Group.query().where('active', true)
|
|
|
|
if (groups.length === 0) {
|
|
this.logger.info('No active groups to index. Add some via `node ace db:seed` or manually.')
|
|
return
|
|
}
|
|
|
|
let conn
|
|
try {
|
|
conn = await this.pool.acquire()
|
|
|
|
for (const group of groups) {
|
|
try {
|
|
const groupInfo: any = await conn.group(group.name)
|
|
|
|
// nntp-js returns article numbers as strings. We must parse them to BigInts.
|
|
const firstArticle = BigInt(groupInfo.first)
|
|
const lastArticle = BigInt(groupInfo.last)
|
|
|
|
// lastIndexedId from the database should also be treated as a BigInt.
|
|
const lastIndexed = group.lastIndexedId ? BigInt(group.lastIndexedId) : null
|
|
|
|
const startId = lastIndexed ? lastIndexed + 1n : firstArticle
|
|
|
|
if (startId > lastArticle) {
|
|
this.logger.info(`No new headers for group ${group.name}.`)
|
|
continue
|
|
}
|
|
|
|
const BATCH_SIZE = 100000n
|
|
const proposedEndId = startId + BATCH_SIZE - 1n
|
|
const endId = proposedEndId < lastArticle ? proposedEndId : lastArticle
|
|
|
|
this.logger.info(`Queueing fetch job for ${group.name}: articles ${startId} to ${endId}`)
|
|
await this.fetchQueue.add('fetch-headers', {
|
|
groupName: group.name,
|
|
startId: startId.toString(),
|
|
endId: endId.toString(),
|
|
})
|
|
|
|
group.lastIndexedId = endId
|
|
await group.save()
|
|
} catch (err: any) {
|
|
this.logger.error(`Error processing group ${group.name}: ${err.message}`)
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
this.logger.error(`Error in scheduler main loop: ${err.message}`)
|
|
} finally {
|
|
if (conn) {
|
|
this.pool.release(conn)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Run once immediately and then on an interval.
|
|
schedule()
|
|
setInterval(schedule, 60000) // 1 minute
|
|
|
|
// Keep the command running
|
|
await new Promise(() => {})
|
|
}
|
|
}
|