Daan MeijerandClaude Opus 5.5
b2cc75fdd0
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
88 lines
4.4 KiB
TypeScript
88 lines
4.4 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
|
|
import SeedService from '#services/SeedService'
|
|
import QueueService from '#services/QueueService'
|
|
import { FileAggregator } from '#lib/aggregate/index'
|
|
import { toFragment, toSeedFile } from '#lib/seed/wire'
|
|
import { insertLimits, segmentsOf } from '#services/FileStore'
|
|
|
|
function header(subject: string, n: number) {
|
|
return {
|
|
'subject': subject,
|
|
'from': 'seeder <seed@example.invalid>',
|
|
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
|
'message-id': `<seed-${n}-${subject.length}@test>`,
|
|
':bytes': '700000',
|
|
}
|
|
}
|
|
|
|
test.group('SeedService', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
group.teardown(() => QueueService.closeAll())
|
|
|
|
test('stores aggregated files with their release and a completed binary', async ({ assert }) => {
|
|
const aggregator = new FileAggregator<IngestSegment>()
|
|
const subjects = [1, 2].map((i) => `Seed.Test.S01E01.1080p - [1/2] - "Seed.Test.S01E01.1080p.part${i}.mkv" yEnc (%d/3)`)
|
|
const segments = subjects.flatMap((s) => [1, 2, 3].map((n) => segmentFromHeader(header(s.replace('%d', String(n)), n), 'alt.binaries.test', n)!))
|
|
const files = aggregator.add(segments).map(toSeedFile)
|
|
assert.lengthOf(files, 2)
|
|
|
|
const result = await SeedService.store(files)
|
|
assert.include(result, { files: 2, existing: 0, releases: 1 })
|
|
|
|
const stored = await db.from('files').where('filename', 'Seed.Test.S01E01.1080p.part1.mkv').first()
|
|
assert.equal(stored.parts, 3)
|
|
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
|
|
const release = await db.from('releases').where('id', stored.release_id).first()
|
|
assert.equal(release.name, segments[0].release!.name) // named as live indexing would
|
|
// A readable name isn't inspected: no binary (LOG.md 2026-10-06 (11)).
|
|
assert.isNull(await db.from('binaries').where('file_key', stored.file_key).first())
|
|
|
|
// The same files again (a crosspost, a second run): nothing new.
|
|
const again = await SeedService.store(files)
|
|
assert.include(again, { files: 0, existing: 2 })
|
|
})
|
|
|
|
test('an obfuscated file keeps a binary until inspection has fingerprinted it', async ({ assert }) => {
|
|
const aggregator = new FileAggregator<IngestSegment>()
|
|
const subject = 'Xq8r2KdLm3Zw91PqA7zz [1/1] - "Xq8r2KdLm3Zw91PqA7zz.bin" yEnc (%d/2)'
|
|
const segments = [1, 2].map((n) => segmentFromHeader(header(subject.replace('%d', String(n)), n), 'alt.binaries.test', n)!)
|
|
const files = aggregator.add(segments).map(toSeedFile)
|
|
const result = await SeedService.store(files)
|
|
assert.include(result, { files: 1, inspections: 1 })
|
|
const binary = await db.from('binaries').where('file_key', files[0].fileKey).first()
|
|
assert.isTrue(binary.awaiting_inspection)
|
|
assert.isNotNull(binary.file_id)
|
|
})
|
|
|
|
test('stores a batch in several INSERTs', async ({ assert }) => {
|
|
// Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
|
|
const rows = insertLimits.rows
|
|
insertLimits.rows = 50
|
|
try {
|
|
const aggregator = new FileAggregator<IngestSegment>()
|
|
const segments = Array.from({ length: 120 }, (_, i) =>
|
|
segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
|
|
)
|
|
const files = aggregator.add(segments).map(toSeedFile)
|
|
assert.include(await SeedService.store(files), { files: 120 })
|
|
const [{ count }] = await db.from('files').whereLike('filename', 'dense.%').count('* as count')
|
|
assert.equal(Number(count), 120)
|
|
} finally {
|
|
insertLimits.rows = rows
|
|
}
|
|
})
|
|
|
|
test('fragments are kept near the range edges only', async ({ assert }) => {
|
|
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
|
|
const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
|
|
const middle = segmentFromHeader(header(subject('middle.bin'), 2), 'alt.binaries.test', 1_500_000)!
|
|
const kept = await SeedService.storeFragments([toFragment(edge), toFragment(middle)], { start: 1_000_000, end: 2_999_999 })
|
|
assert.equal(kept, 1)
|
|
assert.isOk(await db.from('binaries').where('name', 'edge.bin').first())
|
|
assert.isNotOk(await db.from('binaries').where('name', 'middle.bin').first())
|
|
})
|
|
})
|