implemented a lot of features

app/services/QueueService.ts

@@ -1,29 +1,60 @@
-import { Queue } from 'bullmq'
+import { Queue, type DefaultJobOptions } from 'bullmq'
 import queueConfig from '#config/queue'
+import gcConfig from '#config/gc'
 
+const defaultJobOptions: DefaultJobOptions = {
+  // Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 2000 },
+  removeOnComplete: {
+    age: gcConfig.jobs.completedAgeSeconds,
+    count: gcConfig.jobs.completedMaxCount,
+  },
+  removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
+}
+
+/**
+ * Queues are created (and connect to Redis) on first use. Ace imports every
+ * command at boot, so eager queues would keep one-off commands from exiting.
+ */
 class QueueService {
-  public readonly nntpFetchQueue: Queue
-  public readonly headerQueue: Queue
-  public readonly fileQueue: Queue
-  public readonly bodyQueue: Queue
-  public readonly collectionQueue: Queue
+  private readonly queues = new Map<string, Queue>()
 
-  constructor() {
-    this.nntpFetchQueue = new Queue('nntp-fetch-queue', { connection: queueConfig.connection })
-    this.headerQueue = new Queue('header-queue', { connection: queueConfig.connection })
-    this.fileQueue = new Queue('file-queue', { connection: queueConfig.connection })
-    this.bodyQueue = new Queue('body-queue', { connection: queueConfig.connection })
-    this.collectionQueue = new Queue('collection-queue', { connection: queueConfig.connection })
+  private queue(name: string): Queue {
+    let queue = this.queues.get(name)
+    if (!queue) {
+      queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
+      this.queues.set(name, queue)
+    }
+    return queue
   }
 
+  get nntpFetchQueue() {
+    return this.queue('nntp-fetch-queue')
+  }
+  get fileQueue() {
+    return this.queue('file-queue')
+  }
+  get bodyQueue() {
+    return this.queue('body-queue')
+  }
+  get collectionQueue() {
+    return this.queue('collection-queue')
+  }
+
+  /** Queues no longer used by the pipeline, kept here so GC can empty them. */
+  get legacy(): Queue[] {
+    return [this.queue('header-queue')]
+  }
+
+  get all(): Queue[] {
+    return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue]
+  }
+
+  /** Closes the queues that were opened. */
   async closeAll() {
-    await Promise.all([
-      this.nntpFetchQueue.close(),
-      this.headerQueue.close(),
-      this.fileQueue.close(),
-      this.bodyQueue.close(),
-      this.collectionQueue.close(),
-    ])
+    await Promise.all([...this.queues.values()].map((queue) => queue.close()))
+    this.queues.clear()
   }
 }