implemented a lot of features

commands/IndexScheduler.ts

@@ -3,6 +3,10 @@ import { CommandOptions } from '@adonisjs/core/types/ace'
 import Group from '#models/group'
 import NntpService from '#services/NntpService'
 import QueueService from '#services/QueueService'
+import indexerConfig from '#config/indexer'
+import Shutdown from '#services/Shutdown'
+import FetchRangeService from '#services/FetchRangeService'
+import SplitService from '#services/SplitService'
 
 export default class IndexScheduler extends BaseCommand {
   public static commandName = 'index:scheduler'
@@ -13,7 +17,6 @@ export default class IndexScheduler extends BaseCommand {
   }
 
   private pool = NntpService
-  private fetchQueue = QueueService.nntpFetchQueue
 
   public async run() {
     this.logger.info('Scheduler started. Awaiting tasks...')
@@ -28,58 +31,89 @@ export default class IndexScheduler extends BaseCommand {
       }
 
       let conn
+      let failed = false
       try {
         conn = await this.pool.acquire()
 
         for (const group of groups) {
           try {
-            const groupInfo: any = await conn.group(group.name)
-            
-            // nntp-js returns article numbers as strings. We must parse them to BigInts.
-            const firstArticle = BigInt(groupInfo.first)
-            const lastArticle = BigInt(groupInfo.last)
+            const { first: firstArticle, last: lastArticle } = await this.pool.selectGroup(
+              conn,
+              group.name
+            )
 
-            // lastIndexedId from the database should also be treated as a BigInt.
-            const lastIndexed = group.lastIndexedId ? BigInt(group.lastIndexedId) : null
+            await FetchRangeService.requeue(group)
 
-            const startId = lastIndexed ? lastIndexed + 1n : firstArticle
+            // Postgres returns bigint columns as strings.
+            const lastQueued = BigInt(group.lastQueuedId ?? 0)
+            const { batchSize, maxBatchesPerTick, initialBackfill } = indexerConfig
+            let startId =
+              lastQueued === 0n ? lastArticle - initialBackfill + 1n : lastQueued + 1n
+            if (startId < firstArticle) startId = firstArticle
 
             if (startId > lastArticle) {
+              await FetchRangeService.advanceWatermark(group)
               this.logger.info(`No new headers for group ${group.name}.`)
               continue
             }
 
-            const BATCH_SIZE = 100000n
-            const proposedEndId = startId + BATCH_SIZE - 1n
-            const endId = proposedEndId < lastArticle ? proposedEndId : lastArticle
+            let batches = 0
+            let endId = startId - 1n
+            while (endId < lastArticle && batches < maxBatchesPerTick) {
+              const batchStart = endId + 1n
+              endId = batchStart + batchSize - 1n < lastArticle ? batchStart + batchSize - 1n : lastArticle
+              await FetchRangeService.queue(group, batchStart, endId)
+              batches++
+            }
 
-            this.logger.info(`Queueing fetch job for ${group.name}: articles ${startId} to ${endId}`)
-            await this.fetchQueue.add('fetch-headers', {
-              groupName: group.name,
-              startId: startId.toString(),
-              endId: endId.toString(),
-            })
+            this.logger.info(
+              `Queued ${batches} fetch job(s) for ${group.name}: articles ${startId} to ${endId}` +
+                (endId < lastArticle ? ` (${lastArticle - endId} left for next tick)` : '')
+            )
 
-            group.lastIndexedId = endId
+            group.lastQueuedId = endId
             await group.save()
+            const watermark = await FetchRangeService.advanceWatermark(group)
+            if (endId - watermark > batchSize * BigInt(maxBatchesPerTick) * 3n) {
+              this.logger.warning(
+                `${group.name}: indexed up to ${watermark}, ${endId - watermark} articles behind the queue.`
+              )
+            }
           } catch (err: any) {
+            failed = true
             this.logger.error(`Error processing group ${group.name}: ${err.message}`)
           }
         }
       } catch (err: any) {
+        failed = true
         this.logger.error(`Error in scheduler main loop: ${err.message}`)
       } finally {
         if (conn) {
-          this.pool.release(conn)
+          this.pool.release(conn, failed)
         }
       }
     }
 
-    // Run once immediately and then on an interval.
-    schedule()
-    setInterval(schedule, 60000) // 1 minute
+    const split = async () => {
+      try {
+        const result = await SplitService.splitPending()
+        if (result.parents > 0) {
+          this.logger.info(
+            `Split ${result.parents} conflicted binaries: ${result.volumes} verified volumes, ${result.completed} completed.`
+          )
+        }
+      } catch (err: any) {
+        this.logger.error(`Error splitting conflicted binaries: ${err.message}`)
+      }
+    }
 
-    // Keep the command running
-    await new Promise(() => {})
+    // Run immediately, then wait a full interval after each run so runs never overlap.
+    while (!Shutdown.requested) {
+      await schedule()
+      await split()
+      await Shutdown.sleep(indexerConfig.intervalMs)
+    }
+    await this.pool.shutdown()
+    await QueueService.closeAll()
   }
 }