Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
81 lines
3.6 KiB
TypeScript
81 lines
3.6 KiB
TypeScript
import { BaseCommand } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import { Worker } from 'bullmq'
|
|
import queueConfig from '#config/queue'
|
|
import env from '#start/env'
|
|
import Shutdown from '#services/Shutdown'
|
|
import QueueService from '#services/QueueService'
|
|
import NntpService from '#services/NntpService'
|
|
import ContentService from '#services/ContentService'
|
|
import File from '#models/file'
|
|
|
|
export default class CollectionWorker extends BaseCommand {
|
|
public static commandName = 'worker:collection'
|
|
public static description = 'Starts a worker to process file collections (e.g., RAR archives).'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
public async run() {
|
|
this.logger.info('Starting collection worker...')
|
|
|
|
const worker = new Worker('collection-queue', async (job) => {
|
|
const { fileId } = job.data
|
|
this.logger.debug(`Processing file ID ${fileId} for collection.`)
|
|
|
|
const file = await File.find(fileId)
|
|
if (!file) {
|
|
this.logger.error(`File with ID ${fileId} not found in the database.`)
|
|
return
|
|
}
|
|
|
|
try {
|
|
const content = await ContentService.inspect(fileId)
|
|
if (content?.par2) {
|
|
const { par2 } = content
|
|
this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
|
|
} else if (content?.archive) {
|
|
const { info, releaseRenamed, grouped } = content.archive
|
|
const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
|
|
.filter(Boolean)
|
|
.join(', ')
|
|
this.logger.info(
|
|
`Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
|
|
(grouped ? ', grouped into a release' : '') +
|
|
(releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
|
|
)
|
|
}
|
|
if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`)
|
|
if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`)
|
|
if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`)
|
|
if (content?.yencName) this.logger.info(`"${file.filename}" is "${content.yencName}" (yEnc name).`)
|
|
if (content?.nicovideo) this.logger.info(`"${file.filename}" → "${content.nicovideo}".`)
|
|
if (content?.sfvMoved) this.logger.info(`SFV "${file.filename}" moved ${content.sfvMoved} files into its release.`)
|
|
if (content?.contentType === 'yencbin') this.logger.info(`"${file.filename}" is a yEncBin encrypted container.`)
|
|
if (content?.nzb) {
|
|
const n = content.nzb
|
|
this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
|
|
}
|
|
} catch (error: any) {
|
|
this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
|
|
}
|
|
}, {
|
|
connection: queueConfig.connection,
|
|
// Each job holds one NNTP connection while downloading; keep <= the pool size.
|
|
concurrency: env.get('COLLECTION_CONCURRENCY', 1),
|
|
})
|
|
|
|
worker.on('failed', (job, err) => {
|
|
this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
|
|
})
|
|
|
|
this.logger.info('Collection worker started and listening for jobs.')
|
|
await Shutdown.wait()
|
|
this.logger.info('Shutting down: finishing active jobs...')
|
|
await worker.close()
|
|
await NntpService.shutdown()
|
|
await QueueService.closeAll()
|
|
}
|
|
}
|