import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import IngestService, { type IngestSegment } from '#services/IngestService' import BinaryFileService from '#services/BinaryFileService' import QueueService from '#services/QueueService' import FileStore, { segmentsOf } from '#services/FileStore' import { fileKey } from '#lib/subject/index' import ContentService from '#services/ContentService' function segment(name: string, number: number, total = 3): IngestSegment { const poster = 'batch-test@example.invalid' const subject = `"${name}" yEnc` return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] } } const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first() test.group('Binaries to files', (group) => { group.each.setup(() => testUtils.db().withGlobalTransaction()) group.teardown(() => QueueService.closeAll()) test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => { await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)]) // The same article again (a crosspost): not counted twice. await IngestService.ingest([segment('a.bin', 2)]) assert.equal((await binary('a.bin')).part_count, 2) assert.isNull((await binary('a.bin')).completed_at) const completed = await IngestService.ingest([segment('a.bin', 3)]) assert.equal((await binary('a.bin')).part_count, 3) assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey]) }) test('a binary from before the count is counted exactly on its next part', async ({ assert }) => { await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)]) await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null }) const completed = await IngestService.ingest([segment('b.bin', 3)]) assert.equal((await binary('b.bin')).part_count, 3) assert.lengthOf(completed, 1) }) test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => { await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)])) await IngestService.ingest([segment('half.bin', 1)]) // known.bin is a file already (e.g. from an NZB), found by its first segment. const now = new Date() const [known] = await FileStore.insert([ { filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now }, ]) const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id)) const result = await BinaryFileService.store(ids) assert.include(result, { saved: 1, existing: 1 }) assert.deepEqual(result.incomplete, [ids[2]]) const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first() assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3]) assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id)) assert.equal(Number((await binary('known.bin')).file_id), known.id) // Header parts are in the file's segment list now. const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n') assert.equal(Number(n), 0) // The retry: done ones are skipped, the incomplete one again incomplete. const again = await BinaryFileService.store(ids) assert.include(again, { saved: 0, skipped: 2 }) assert.deepEqual(again.incomplete, [ids[2]]) }) test('a queued obfuscated file keeps its binary until inspection fingerprints it', async ({ assert }) => { const name = 'Zk3m9QpL2xW8rT5vN1bYq.bin' await IngestService.ingest([1, 2, 3].map((n) => segment(name, n))) const b = await binary(name) const result = await BinaryFileService.store([Number(b.id)]) assert.include(result, { saved: 1, inspections: 1 }) const waiting = await binary(name) assert.isTrue(waiting.awaiting_inspection) // Inspection stores the fingerprint and clears the flag. const task = { kind: 'inspect' as const, fileId: Number(waiting.file_id), name, segments: [], sevenZip: null } await ContentService.apply(task, { size: 300, contentType: null, md5_16k: 'a'.repeat(32), yencName: name }) const inspected = await binary(name) assert.isNull(inspected.awaiting_inspection) assert.equal(inspected.md5_16k, 'a'.repeat(32)) }) })