Daan MeijerandClaude Opus 5.5
f4ded062df
Inspection lists only the entries whose headers are in the first segment of a first volume. A walk hops from header to header, fetching one segment per entry and volume, in its own queue (one job per collection worker) on NNTP connections inspection isn't waiting for. content:walk-archives queues releases inspected earlier (CONCEPT D40). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
66 lines
1.9 KiB
TypeScript
66 lines
1.9 KiB
TypeScript
import { Queue, type DefaultJobOptions } from 'bullmq'
|
|
import queueConfig from '#config/queue'
|
|
import gcConfig from '#config/gc'
|
|
|
|
const defaultJobOptions: DefaultJobOptions = {
|
|
// Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
|
|
attempts: 3,
|
|
backoff: { type: 'exponential', delay: 2000 },
|
|
removeOnComplete: {
|
|
age: gcConfig.jobs.completedAgeSeconds,
|
|
count: gcConfig.jobs.completedMaxCount,
|
|
},
|
|
removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
|
|
}
|
|
|
|
/**
|
|
* Queues are created (and connect to Redis) on first use. Ace imports every
|
|
* command at boot, so eager queues would keep one-off commands from exiting.
|
|
*/
|
|
class QueueService {
|
|
private readonly queues = new Map<string, Queue>()
|
|
|
|
private queue(name: string): Queue {
|
|
let queue = this.queues.get(name)
|
|
if (!queue) {
|
|
queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
|
|
this.queues.set(name, queue)
|
|
}
|
|
return queue
|
|
}
|
|
|
|
get nntpFetchQueue() {
|
|
return this.queue('nntp-fetch-queue')
|
|
}
|
|
get fileQueue() {
|
|
return this.queue('file-queue')
|
|
}
|
|
get bodyQueue() {
|
|
return this.queue('body-queue')
|
|
}
|
|
get collectionQueue() {
|
|
return this.queue('collection-queue')
|
|
}
|
|
/** Complete archive listings, at low priority (ArchiveWalkService). */
|
|
get archiveWalkQueue() {
|
|
return this.queue('archive-walk-queue')
|
|
}
|
|
|
|
/** Queues no longer used by the pipeline, kept here so GC can empty them. */
|
|
get legacy(): Queue[] {
|
|
return [this.queue('header-queue')]
|
|
}
|
|
|
|
get all(): Queue[] {
|
|
return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue, this.archiveWalkQueue]
|
|
}
|
|
|
|
/** Closes the queues that were opened. */
|
|
async closeAll() {
|
|
await Promise.all([...this.queues.values()].map((queue) => queue.close()))
|
|
this.queues.clear()
|
|
}
|
|
}
|
|
|
|
export default new QueueService()
|