diff --git a/app/services/ContentService.ts b/app/services/ContentService.ts index 05c8171..21c05a2 100644 --- a/app/services/ContentService.ts +++ b/app/services/ContentService.ts @@ -156,6 +156,11 @@ class ContentService { const binary = await db.from('binaries').where('file_id', task.fileId).first() const { contentType } = probe await db.from('files').where('id', file.id).update({ content_type: contentType }) + if (contentType === 'missing') { + // Nothing to read; the binary needn't wait for inspection any more. + if (binary) await db.from('binaries').where('id', binary.id).update({ awaiting_inspection: null }) + return result + } // Fingerprint, so par2 sets (now or later) can name this file exactly. if (binary) { @@ -268,6 +273,7 @@ export function describeContent(filename: string, content: ContentResult | null) ) } if (content?.named) lines.push(`"${filename}" got its real name from par2.`) + if (content?.contentType === 'missing') lines.push(`"${filename}": the server no longer has its first article.`) if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`) if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`) if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`) diff --git a/app/services/RemoteContentService.ts b/app/services/RemoteContentService.ts index 02182d1..0dc52e4 100644 --- a/app/services/RemoteContentService.ts +++ b/app/services/RemoteContentService.ts @@ -7,7 +7,7 @@ import QueueService from '#services/QueueService' import NntpService from '#services/NntpService' import ContentService, { describeContent } from '#services/ContentService' import ArchiveWalkService from '#services/ArchiveWalkService' -import type { ContentTask, InspectProbe, InspectTask, WalkProbe } from '#lib/content/probe' +import { isMissingArticle, type ContentTask, type InspectProbe, type InspectTask, type WalkProbe } from '#lib/content/probe' /** Inspection first; archive walks (D40) only when it has nothing waiting. */ const QUEUES = ['collection-queue', 'archive-walk-queue'] as const @@ -90,6 +90,11 @@ class RemoteContentService { /** The probe failed on the node (e.g. a missing article): retried like a local failure. */ async fail(queue: QueueName, jobId: string, token: string, error: string) { const job = await this.job(queue, jobId) + // A missing first article from a node that predates the probe's own handling: a result, + // not something to retry (each try costs Eweka ~450 ms; LOG.md 2026-10-06 (14)). + if (queue === 'collection-queue' && job.data.remote && isMissingArticle(error)) { + return this.finish(queue, jobId, token, { size: 0, contentType: 'missing', md5_16k: null, yencName: '' }) + } await this.locked(() => job.moveToFailed(new Error(`node: ${error}`), token, false)) } diff --git a/lib/content/probe.ts b/lib/content/probe.ts index 6b32173..aa55d4c 100644 --- a/lib/content/probe.ts +++ b/lib/content/probe.ts @@ -83,6 +83,11 @@ export function chunkMd5(data: Buffer, fileSize: number): string | undefined { return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex') } +/** A "no such article" answer: 430 from the news server, 410 from the seed relay. */ +export function isMissingArticle(error: unknown): boolean { + return /(^|[^0-9])(430|410)([^0-9]|$)/.test(String((error as any)?.message ?? error)) && !/timed out|no answer/i.test(String((error as any)?.message)) +} + /** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */ export function decodeBody(body: Buffer): DecodedSegment { try { @@ -132,7 +137,17 @@ export async function probeInspection( fetch: BodyFetcher, warn: (message: string) => void = () => {} ): Promise { - const segment = decodeBody(await fetch(task.segments[0].id)) + let body: Buffer + try { + body = await fetch(task.segments[0].id) + } catch (error: any) { + // The server no longer has it (430; 410 through the seed relay), typically a takedown. A final + // answer, and an expensive one: Eweka takes ~450 ms per missing article, so it isn't retried + // (LOG.md 2026-10-06 (14)). + if (isMissingArticle(error)) return { size: 0, contentType: 'missing', md5_16k: null, yencName: '' } + throw error + } + const segment = decodeBody(body) const size = segment.meta.begin.size const contentType = sniffContentType(segment.data) const probe: InspectProbe = { diff --git a/lib/content/sniff.ts b/lib/content/sniff.ts index ccb6381..ebd15b9 100644 --- a/lib/content/sniff.ts +++ b/lib/content/sniff.ts @@ -6,6 +6,8 @@ * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy. */ export type ContentType = + /** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */ + | 'missing' | 'nzbe' | 'par2' | 'rar4' diff --git a/tests/functional/seed_api.spec.ts b/tests/functional/seed_api.spec.ts index cd4f5c5..edfc7ea 100644 --- a/tests/functional/seed_api.spec.ts +++ b/tests/functional/seed_api.spec.ts @@ -234,11 +234,28 @@ test.group('Seed API', (group) => { ;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200) const again = await lease() assert.equal(again.jobId, first.jobId) - ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: '430 no such article' }).headers(auth)).assertStatus(200) + ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: 'NNTP BODY: no answer within 120 s' }).headers(auth)).assertStatus(200) // Retried later (backoff), like a local failure. assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState()) }) + test('a missing first article reported by a node is recorded, not retried', async ({ client, assert }) => { + await QueueService.collectionQueue.drain() + await QueueService.archiveWalkQueue.drain() + const token = await SeedOrchestrator.issueToken('inspect-node-3') + const auth = { authorization: `Bearer ${token}` } + const now = new Date() + const [file] = await FileStore.insert([ + { filename: 'Wq7n3LpX9vT2rK8mZ5cYbA.bin', file_key: 'inspect-node-file-3'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'gone@test', size: 1 }], created_at: now, updated_at: now }, + ]) + await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` }) + const job = (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0] + const error = 'Seed API /api/v1/seed/articles?id=gone%40test: 410 {"error":"No such article"}' + ;(await client.post('/api/v1/seed/inspect/failed').json({ queue: job.queue, jobId: job.jobId, token: job.token, error }).headers(auth)).assertStatus(200) + assert.equal((await db.from('files').where('id', file.id).first()).content_type, 'missing') + assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed') + }) + test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => { const token = await SeedOrchestrator.issueToken('late-node') const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id') diff --git a/tests/unit/probe.spec.ts b/tests/unit/probe.spec.ts index 1b306a0..5616b43 100644 --- a/tests/unit/probe.spec.ts +++ b/tests/unit/probe.spec.ts @@ -71,9 +71,16 @@ test.group('content probe', () => { assert.lengthOf(probe.archive!.entries, 0) }) - test('a missing article fails the probe (the job is retried)', async ({ assert }) => { + test('a missing first article is a result, not a failure (no retries)', async ({ assert }) => { const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1)) task.segments[0].id = 'missing@test' - await assert.rejects(() => probeInspection(task, fetch), /430/) + assert.include(await probeInspection(task, fetch), { contentType: 'missing' }) + // Through the seed relay: 410. + const relayed = await probeInspection(task, async () => { + throw new Error('Seed API /api/v1/seed/articles?id=x: 410 {"error":"No such article"}') + }) + assert.equal(relayed.contentType, 'missing') + // Anything else still fails (and is retried). + await assert.rejects(() => probeInspection(task, async () => { throw new Error('NNTP BODY: no answer within 120 s') }), /no answer/) }) })