Daan MeijerandClaude Opus 5.5
b2cc75fdd0
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
4.7 KiB
TypeScript
87 lines
4.7 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import IngestService, { type IngestSegment } from '#services/IngestService'
|
|
import BinaryFileService from '#services/BinaryFileService'
|
|
import QueueService from '#services/QueueService'
|
|
import FileStore, { segmentsOf } from '#services/FileStore'
|
|
import { fileKey } from '#lib/subject/index'
|
|
import ContentService from '#services/ContentService'
|
|
|
|
function segment(name: string, number: number, total = 3): IngestSegment {
|
|
const poster = 'batch-test@example.invalid'
|
|
const subject = `"${name}" yEnc`
|
|
return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] }
|
|
}
|
|
|
|
const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first()
|
|
|
|
test.group('Binaries to files', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
group.teardown(() => QueueService.closeAll())
|
|
|
|
test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => {
|
|
await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)])
|
|
// The same article again (a crosspost): not counted twice.
|
|
await IngestService.ingest([segment('a.bin', 2)])
|
|
assert.equal((await binary('a.bin')).part_count, 2)
|
|
assert.isNull((await binary('a.bin')).completed_at)
|
|
const completed = await IngestService.ingest([segment('a.bin', 3)])
|
|
assert.equal((await binary('a.bin')).part_count, 3)
|
|
assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey])
|
|
})
|
|
|
|
test('a binary from before the count is counted exactly on its next part', async ({ assert }) => {
|
|
await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)])
|
|
await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null })
|
|
const completed = await IngestService.ingest([segment('b.bin', 3)])
|
|
assert.equal((await binary('b.bin')).part_count, 3)
|
|
assert.lengthOf(completed, 1)
|
|
})
|
|
|
|
test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => {
|
|
await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)]))
|
|
await IngestService.ingest([segment('half.bin', 1)])
|
|
// known.bin is a file already (e.g. from an NZB), found by its first segment.
|
|
const now = new Date()
|
|
const [known] = await FileStore.insert([
|
|
{ filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now },
|
|
])
|
|
const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id))
|
|
|
|
const result = await BinaryFileService.store(ids)
|
|
assert.include(result, { saved: 1, existing: 1 })
|
|
assert.deepEqual(result.incomplete, [ids[2]])
|
|
|
|
const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first()
|
|
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
|
|
assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id))
|
|
assert.equal(Number((await binary('known.bin')).file_id), known.id)
|
|
// Header parts are in the file's segment list now.
|
|
const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n')
|
|
assert.equal(Number(n), 0)
|
|
|
|
// The retry: done ones are skipped, the incomplete one again incomplete.
|
|
const again = await BinaryFileService.store(ids)
|
|
assert.include(again, { saved: 0, skipped: 2 })
|
|
assert.deepEqual(again.incomplete, [ids[2]])
|
|
})
|
|
|
|
test('a queued obfuscated file keeps its binary until inspection fingerprints it', async ({ assert }) => {
|
|
const name = 'Zk3m9QpL2xW8rT5vN1bYq.bin'
|
|
await IngestService.ingest([1, 2, 3].map((n) => segment(name, n)))
|
|
const b = await binary(name)
|
|
const result = await BinaryFileService.store([Number(b.id)])
|
|
assert.include(result, { saved: 1, inspections: 1 })
|
|
const waiting = await binary(name)
|
|
assert.isTrue(waiting.awaiting_inspection)
|
|
|
|
// Inspection stores the fingerprint and clears the flag.
|
|
const task = { kind: 'inspect' as const, fileId: Number(waiting.file_id), name, segments: [], sevenZip: null }
|
|
await ContentService.apply(task, { size: 300, contentType: null, md5_16k: 'a'.repeat(32), yencName: name })
|
|
const inspected = await binary(name)
|
|
assert.isNull(inspected.awaiting_inspection)
|
|
assert.equal(inspected.md5_16k, 'a'.repeat(32))
|
|
})
|
|
})
|