Seed batches: test chunked inserts with a lowered chunk size
The 6,000-file test took 29 s and failed CI; 120 files in chunks of 50 cover the same path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
953bbe9e39
commit
9ae15ed26f
3 files changed
+25
-21
No files matched your search
@@ -36,8 +36,8 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
|
||||
return stored ? unpackSegments(stored) : []
|
||||
}
|
||||
|
||||
/** Rows per INSERT, well below Postgres' 65,535 bind parameters. */
|
||||
export const INSERT_ROWS = 2000
|
||||
/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
|
||||
export const insertLimits = { rows: 2000 }
|
||||
|
||||
class FileStore {
|
||||
/**
|
||||
@@ -71,8 +71,8 @@ class FileStore {
|
||||
if (rows.length === 0) return []
|
||||
// A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
|
||||
const inserted: { id: number; filename: string }[] = []
|
||||
for (let i = 0; i < rows.length; i += INSERT_ROWS) {
|
||||
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + INSERT_ROWS)).onConflict().ignore().returning(['id', 'filename'])
|
||||
for (let i = 0; i < rows.length; i += insertLimits.rows) {
|
||||
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename'])
|
||||
for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename })
|
||||
}
|
||||
return inserted
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import FileStore, { INSERT_ROWS } from '#services/FileStore'
|
||||
import FileStore, { insertLimits } from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import QueueService from '#services/QueueService'
|
||||
@@ -78,9 +78,9 @@ class SeedService {
|
||||
file_id: fileIds.get(file.fileKey) ?? null,
|
||||
}))
|
||||
binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
for (let i = 0; i < binaries.length; i += INSERT_ROWS) {
|
||||
for (let i = 0; i < binaries.length; i += insertLimits.rows) {
|
||||
await knex('binaries')
|
||||
.insert(binaries.slice(i, i + INSERT_ROWS))
|
||||
.insert(binaries.slice(i, i + insertLimits.rows))
|
||||
.onConflict('file_key')
|
||||
.merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user