Seed batches: test chunked inserts with a lowered chunk size
Test and deploy / test (push) Successful in 1m57s
Test and deploy / deploy (push) Successful in 4m7s

The 6,000-file test took 29 s and failed CI; 120 files in chunks of 50
cover the same path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 00:20:17 +02:00
1 parent 953bbe9e39
commit 9ae15ed26f
3 files changed
+25 -21

No files matched your search

+18 -14
View File
@@ -6,7 +6,7 @@ import SeedService from '#services/SeedService'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile } from '#lib/seed/wire'
import { segmentsOf } from '#services/FileStore'
import { insertLimits, segmentsOf } from '#services/FileStore'
function header(subject: string, n: number) {
return {
@@ -46,19 +46,23 @@ test.group('SeedService', (group) => {
assert.include(again, { files: 0, existing: 2 })
})
test('stores batches of more files than one INSERT can take', async ({ assert }) => {
// 12 columns per row: over 5,461 rows exceed Postgres' 65,535 bind parameters.
const aggregator = new FileAggregator<IngestSegment>()
const segments = Array.from({ length: 6000 }, (_, i) =>
segmentFromHeader(header(`Dense.Upload - [${i + 1}/6000] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
)
const files = aggregator.add(segments).map(toSeedFile)
assert.lengthOf(files, 6000)
const result = await SeedService.store(files)
assert.include(result, { files: 6000 })
const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
assert.equal(Number(count), 6000)
}).timeout(60_000)
test('stores a batch in several INSERTs', async ({ assert }) => {
// Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
const rows = insertLimits.rows
insertLimits.rows = 50
try {
const aggregator = new FileAggregator<IngestSegment>()
const segments = Array.from({ length: 120 }, (_, i) =>
segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
)
const files = aggregator.add(segments).map(toSeedFile)
assert.include(await SeedService.store(files), { files: 120 })
const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
assert.equal(Number(count), 120)
} finally {
insertLimits.rows = rows
}
})
test('fragments are kept near the range edges only', async ({ assert }) => {
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`