Daan MeijerandClaude Opus 5.5
693e6c3244
- seed:plan plans every part of a group's history no range covers (it started below the lowest known range and left a.b.ath's 13 most recent months out); seed_ranges.priority orders leases (seed:plan --priority). - binaries.part_count, kept by the parts insert in the same statement: claimCompleted compares it instead of counting parts every batch. - file_keys: 64-bit hashes of all file keys; FileStore.insert asks it before probing every files partition. - worker:file stores 100 completed binaries per job with set-based queries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
55 lines
2.2 KiB
TypeScript
55 lines
2.2 KiB
TypeScript
import { BaseCommand } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import { Worker } from 'bullmq'
|
|
import env from '#start/env'
|
|
import BinaryFileService from '#services/BinaryFileService'
|
|
import queueConfig from '#config/queue'
|
|
import Shutdown from '#services/Shutdown'
|
|
import QueueService from '#services/QueueService'
|
|
|
|
export default class FileWorker extends BaseCommand {
|
|
public static commandName = 'worker:file'
|
|
public static description = 'Starts a worker that turns completed binaries into files.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
public async run() {
|
|
this.logger.info('Starting file worker...')
|
|
|
|
const worker = new Worker('file-queue', async (job) => {
|
|
// A batch of completed binaries ('process-files'), or one from before batching ('process-file').
|
|
const ids: number[] = (job.data.binaryIds ?? [job.data.binaryId]).map(Number)
|
|
const result = await BinaryFileService.store(ids)
|
|
if (result.saved || result.existing) {
|
|
this.logger.info(
|
|
`Files: ${result.saved} saved, ${result.existing} existed already` +
|
|
(result.skipped ? `, ${result.skipped} skipped` : '') +
|
|
(result.inspections ? `, ${result.inspections} queued for inspection` : '') +
|
|
` (${ids.length} binaries).`
|
|
)
|
|
}
|
|
// The rest is stored; the job's retry picks these up (the others are linked now and skipped).
|
|
if (result.incomplete.length > 0) {
|
|
throw new Error(`${result.incomplete.length} of ${ids.length} binaries don't have all their parts yet.`)
|
|
}
|
|
}, {
|
|
connection: queueConfig.connection,
|
|
// One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and
|
|
// the file insert ignores duplicates, so they can run side by side.
|
|
concurrency: env.get('FILE_CONCURRENCY', 8),
|
|
})
|
|
|
|
worker.on('failed', (job, err) => {
|
|
this.logger.error(`File job ${job?.id} failed: ${err.message}`)
|
|
})
|
|
|
|
this.logger.info('File worker started and listening for jobs.')
|
|
await Shutdown.wait()
|
|
this.logger.info('Shutting down: finishing active jobs...')
|
|
await worker.close()
|
|
await QueueService.closeAll()
|
|
}
|
|
}
|