Complete archive listings by walking RAR headers, at low priority
Inspection lists only the entries whose headers are in the first segment of a first volume. A walk hops from header to header, fetching one segment per entry and volume, in its own queue (one job per collection worker) on NNTP connections inspection isn't waiting for. content:walk-archives queues releases inspected earlier (CONCEPT D40). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b7ae8267c6
commit
f4ded062df
17 files changed
+597
-17
No files matched your search
@@ -7,6 +7,7 @@ import Shutdown from '#services/Shutdown'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import File from '#models/file'
|
||||
|
||||
export default class CollectionWorker extends BaseCommand {
|
||||
@@ -70,10 +71,28 @@ export default class CollectionWorker extends BaseCommand {
|
||||
this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
|
||||
})
|
||||
|
||||
// Complete archive listings (D40): background work beside inspection, on the same NNTP pool.
|
||||
const walkConcurrency = env.get('ARCHIVE_WALK_CONCURRENCY', 1)
|
||||
const walker = walkConcurrency > 0
|
||||
? new Worker('archive-walk-queue', async (job) => {
|
||||
const o = await ArchiveWalkService.walk(Number(job.data.releaseId))
|
||||
if (o.skipped) return o
|
||||
this.logger.info(
|
||||
`Archive walk of release ${o.releaseId}: ${o.before} → ${o.entries} entries from ${o.segments} segments` +
|
||||
(o.complete ? '' : ` (stopped: ${o.stopped})`) +
|
||||
(o.renamed ? `, renamed to "${o.renamed}"` : '')
|
||||
)
|
||||
return o
|
||||
}, { connection: queueConfig.connection, concurrency: walkConcurrency })
|
||||
: null
|
||||
walker?.on('failed', (job, err) => {
|
||||
this.logger.error(`Archive walk job ${job?.id} failed: ${err.message}`)
|
||||
})
|
||||
|
||||
this.logger.info('Collection worker started and listening for jobs.')
|
||||
await Shutdown.wait()
|
||||
this.logger.info('Shutting down: finishing active jobs...')
|
||||
await worker.close()
|
||||
await Promise.all([worker.close(), walker?.close()])
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user