Seed batches: insert files and binaries in chunks of 2,000 rows
A batch of a dense range (over ~5,400 files) exceeded Postgres' 65,535 bind parameters per statement, so its post failed with a 500 until the range was given up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f4ded062df
commit
953bbe9e39
3 files changed
+31
-7
No files matched your search
@@ -46,6 +46,20 @@ test.group('SeedService', (group) => {
|
||||
assert.include(again, { files: 0, existing: 2 })
|
||||
})
|
||||
|
||||
test('stores batches of more files than one INSERT can take', async ({ assert }) => {
|
||||
// 12 columns per row: over 5,461 rows exceed Postgres' 65,535 bind parameters.
|
||||
const aggregator = new FileAggregator<IngestSegment>()
|
||||
const segments = Array.from({ length: 6000 }, (_, i) =>
|
||||
segmentFromHeader(header(`Dense.Upload - [${i + 1}/6000] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
|
||||
)
|
||||
const files = aggregator.add(segments).map(toSeedFile)
|
||||
assert.lengthOf(files, 6000)
|
||||
const result = await SeedService.store(files)
|
||||
assert.include(result, { files: 6000 })
|
||||
const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
|
||||
assert.equal(Number(count), 6000)
|
||||
}).timeout(60_000)
|
||||
|
||||
test('fragments are kept near the range edges only', async ({ assert }) => {
|
||||
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
|
||||
const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
|
||||
|
||||
Reference in new issue
Block a user