Daan MeijerandClaude Opus 5.5
8d86f7d489
About 5% of seeded headers are fragments. They went through staging, where they waited 30 minutes for parts that were never coming (the neighbouring range is worked on by another node, often hours later) and expired: the staging table grew to 22 GB and the NVMe part of the database reached its 60 GB limit, which stopped leasing and the backfill. Files cut by a range edge can still be completed by the neighbouring range, so fragments within 50k articles of an edge go straight into binaries and parts; the rest, files with articles missing for good, are dropped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
59 lines
2.9 KiB
TypeScript
59 lines
2.9 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
|
|
import SeedService from '#services/SeedService'
|
|
import QueueService from '#services/QueueService'
|
|
import { FileAggregator } from '#lib/aggregate/index'
|
|
import { toFragment, toSeedFile } from '#lib/seed/wire'
|
|
import { segmentsOf } from '#services/FileStore'
|
|
|
|
function header(subject: string, n: number) {
|
|
return {
|
|
'subject': subject,
|
|
'from': 'seeder <seed@example.invalid>',
|
|
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
|
'message-id': `<seed-${n}-${subject.length}@test>`,
|
|
':bytes': '700000',
|
|
}
|
|
}
|
|
|
|
test.group('SeedService', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
group.teardown(() => QueueService.closeAll())
|
|
|
|
test('stores aggregated files with their release and a completed binary', async ({ assert }) => {
|
|
const aggregator = new FileAggregator<IngestSegment>()
|
|
const subjects = [1, 2].map((i) => `Seed.Test.S01E01.1080p - [1/2] - "Seed.Test.S01E01.1080p.part${i}.mkv" yEnc (%d/3)`)
|
|
const segments = subjects.flatMap((s) => [1, 2, 3].map((n) => segmentFromHeader(header(s.replace('%d', String(n)), n), 'alt.binaries.test', n)!))
|
|
const files = aggregator.add(segments).map(toSeedFile)
|
|
assert.lengthOf(files, 2)
|
|
|
|
const result = await SeedService.store(files)
|
|
assert.include(result, { files: 2, existing: 0, releases: 1 })
|
|
|
|
const stored = await db.from('files').where('filename', 'Seed.Test.S01E01.1080p.part1.mkv').first()
|
|
assert.equal(stored.parts, 3)
|
|
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
|
|
const release = await db.from('releases').where('id', stored.release_id).first()
|
|
assert.equal(release.name, segments[0].release!.name) // named as live indexing would
|
|
const binary = await db.from('binaries').where('file_key', stored.file_key).first()
|
|
assert.equal(Number(binary.file_id), Number(stored.id))
|
|
assert.isNotNull(binary.completed_at)
|
|
|
|
// The same files again (a crosspost, a second run): nothing new.
|
|
const again = await SeedService.store(files)
|
|
assert.include(again, { files: 0, existing: 2 })
|
|
})
|
|
|
|
test('fragments are kept near the range edges only', async ({ assert }) => {
|
|
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
|
|
const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
|
|
const middle = segmentFromHeader(header(subject('middle.bin'), 2), 'alt.binaries.test', 1_500_000)!
|
|
const kept = await SeedService.storeFragments([toFragment(edge), toFragment(middle)], { start: 1_000_000, end: 2_999_999 })
|
|
assert.equal(kept, 1)
|
|
assert.isOk(await db.from('binaries').where('name', 'edge.bin').first())
|
|
assert.isNotOk(await db.from('binaries').where('name', 'middle.bin').first())
|
|
})
|
|
})
|