From 95f1d427dfd374f9ae76569789ddc3d5cf07f4ce Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 4 Oct 2026 09:32:40 +0200 Subject: [PATCH] Inspect fewer files: skip later volumes under random stems Of 5,000 waiting inspection jobs, 3,212 were queued only because their stem looked random, and 2,044 of those were later RAR volumes (xyz.part47.rar, .r00) or par2 recovery volumes (.vol03+04.par2). Their first segment adds nothing: the set is listed from its first volume and named from its index .par2, as for readable names already. Each one cost a ~700 KB download; inspection pulled ~85 MB/s and fell behind the backfill. - isCandidate() rejects later volumes (isLaterVolume) before the random- name rule. - worker:file only queues files that are candidates, so the ~30% that inspection skips anyway no longer take queue slots. Co-Authored-By: Claude Opus 5.5 --- app/services/ContentService.ts | 3 +++ commands/workers/File.ts | 6 +++++- lib/content/archive_names.ts | 12 ++++++++++++ lib/content/index.ts | 2 +- tests/unit/content.spec.ts | 13 ++++++++++++- 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/app/services/ContentService.ts b/app/services/ContentService.ts index d0b47e2..8ae264b 100644 --- a/app/services/ContentService.ts +++ b/app/services/ContentService.ts @@ -8,6 +8,7 @@ import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/inde import NzbImportService, { type NzbImportResult } from '#services/NzbImportService' import { isArchiveStart, + isLaterVolume, NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, @@ -68,6 +69,8 @@ export interface ContentResult { class ContentService { /** Random or missing names, and named first RAR volumes (for contents and password flag). */ isCandidate(name: string) { + // Even under a random stem: the first volume and the index .par2 are inspected instead. + if (isLaterVolume(name)) return false if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true if (isArchiveStart(name)) return true if (/\.nzb$/i.test(name)) return true diff --git a/commands/workers/File.ts b/commands/workers/File.ts index 5d07a1c..33bfb47 100644 --- a/commands/workers/File.ts +++ b/commands/workers/File.ts @@ -8,6 +8,7 @@ import SfvService from '#services/SfvService' import queueConfig from '#config/queue' import Shutdown from '#services/Shutdown' import QueueService from '#services/QueueService' +import ContentService from '#services/ContentService' export default class FileWorker extends BaseCommand { public static commandName = 'worker:file' @@ -94,7 +95,10 @@ export default class FileWorker extends BaseCommand { date: binary.posted_at ? new Date(binary.posted_at).getTime() : null, release_id: binary.release_id, }) - await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` }) + // Files inspection would skip anyway (later volumes, readable media names) don't take a queue slot. + if (ContentService.isCandidate(binary.name)) { + await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` }) + } }, { connection: queueConfig.connection, // One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and diff --git a/lib/content/archive_names.ts b/lib/content/archive_names.ts index 60a0841..2727ef9 100644 --- a/lib/content/archive_names.ts +++ b/lib/content/archive_names.ts @@ -11,3 +11,15 @@ export function isArchiveStart(name: string): boolean { if (volume === 0 || volume === 1) return true return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name)) } + +/** + * A later RAR volume (.part02.rar, .r00) or a par2 recovery volume + * (.vol03+04.par2): the volume number is readable even under a random stem. + * The set is listed from its first volume and named from its index .par2, + * so the first segment of these says nothing new (LOG.md 2026-10-04). + */ +export function isLaterVolume(name: string): boolean { + const part = /\.part(\d+)\.rar$/i.exec(name) + if (part) return Number(part[1]) > 1 + return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name) +} diff --git a/lib/content/index.ts b/lib/content/index.ts index eb8702b..adbb5e7 100644 --- a/lib/content/index.ts +++ b/lib/content/index.ts @@ -2,6 +2,6 @@ * Content recognition by magic bytes, independent of AdonisJS. */ export { sniffContentType, type ContentType } from './sniff.js' -export { isArchiveStart } from './archive_names.js' +export { isArchiveStart, isLaterVolume } from './archive_names.js' export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js' export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js' diff --git a/tests/unit/content.spec.ts b/tests/unit/content.spec.ts index 2a913ce..f99f2d6 100644 --- a/tests/unit/content.spec.ts +++ b/tests/unit/content.spec.ts @@ -1,6 +1,6 @@ import { test } from '@japa/runner' import yencode from 'yencode' -import { sniffContentType } from '#lib/content/index' +import { isLaterVolume, sniffContentType } from '#lib/content/index' import { crc32Hex, decodeSegment, YencError } from '#lib/yenc/index' test.group('content / sniffContentType', () => { @@ -52,3 +52,14 @@ test.group('yenc / lenientSize', () => { assert.isTrue(decodeSegment(body, { lenientSize: true }).data.equals(data)) }) }) + +test.group('content / isLaterVolume', () => { + test('recognises later RAR volumes and par2 recovery volumes, also under a random stem', ({ assert }) => { + for (const name of ['P8vttYaPec1mL4nDZHPZo7Nd6iEWj8lP.part47.rar', 'Show.S01E01.part002.rar', 'abc.r00', 'abc.r123', 'x7Kq9.vol03+04.par2', 'Show.vol127-128.par2']) { + assert.isTrue(isLaterVolume(name), name) + } + for (const name of ['P8vttYaPec1mL4nDZHPZo7Nd6iEWj8lP.part01.rar', 'abc.part1.rar', 'abc.rar', 'x7Kq9.par2', '6e13186238ab54ca498e46d2d77da099', 'abc.7z.002', 'abc.mkv']) { + assert.isFalse(isLaterVolume(name), name) + } + }) +})