Daan MeijerandClaude Opus 5.5
693e6c3244
- seed:plan plans every part of a group's history no range covers (it started below the lowest known range and left a.b.ath's 13 most recent months out); seed_ranges.priority orders leases (seed:plan --priority). - binaries.part_count, kept by the parts insert in the same statement: claimCompleted compares it instead of counting parts every batch. - file_keys: 64-bit hashes of all file keys; FileStore.insert asks it before probing every files partition. - worker:file stores 100 completed binaries per job with set-based queries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69 lines
3.7 KiB
TypeScript
69 lines
3.7 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import IngestService, { type IngestSegment } from '#services/IngestService'
|
|
import BinaryFileService from '#services/BinaryFileService'
|
|
import QueueService from '#services/QueueService'
|
|
import FileStore, { segmentsOf } from '#services/FileStore'
|
|
import { fileKey } from '#lib/subject/index'
|
|
|
|
function segment(name: string, number: number, total = 3): IngestSegment {
|
|
const poster = 'batch-test@example.invalid'
|
|
const subject = `"${name}" yEnc`
|
|
return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] }
|
|
}
|
|
|
|
const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first()
|
|
|
|
test.group('Binaries to files', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
group.teardown(() => QueueService.closeAll())
|
|
|
|
test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => {
|
|
await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)])
|
|
// The same article again (a crosspost): not counted twice.
|
|
await IngestService.ingest([segment('a.bin', 2)])
|
|
assert.equal((await binary('a.bin')).part_count, 2)
|
|
assert.isNull((await binary('a.bin')).completed_at)
|
|
const completed = await IngestService.ingest([segment('a.bin', 3)])
|
|
assert.equal((await binary('a.bin')).part_count, 3)
|
|
assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey])
|
|
})
|
|
|
|
test('a binary from before the count is counted exactly on its next part', async ({ assert }) => {
|
|
await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)])
|
|
await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null })
|
|
const completed = await IngestService.ingest([segment('b.bin', 3)])
|
|
assert.equal((await binary('b.bin')).part_count, 3)
|
|
assert.lengthOf(completed, 1)
|
|
})
|
|
|
|
test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => {
|
|
await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)]))
|
|
await IngestService.ingest([segment('half.bin', 1)])
|
|
// known.bin is a file already (e.g. from an NZB), found by its first segment.
|
|
const now = new Date()
|
|
const [known] = await FileStore.insert([
|
|
{ filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now },
|
|
])
|
|
const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id))
|
|
|
|
const result = await BinaryFileService.store(ids)
|
|
assert.include(result, { saved: 1, existing: 1 })
|
|
assert.deepEqual(result.incomplete, [ids[2]])
|
|
|
|
const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first()
|
|
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
|
|
assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id))
|
|
assert.equal(Number((await binary('known.bin')).file_id), known.id)
|
|
// Header parts are in the file's segment list now.
|
|
const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n')
|
|
assert.equal(Number(n), 0)
|
|
|
|
// The retry: done ones are skipped, the incomplete one again incomplete.
|
|
const again = await BinaryFileService.store(ids)
|
|
assert.include(again, { saved: 0, skipped: 2 })
|
|
assert.deepEqual(again.incomplete, [ids[2]])
|
|
})
|
|
})
|