Seed batches: insert files and binaries in chunks of 2,000 rows
A batch of a dense range (over ~5,400 files) exceeded Postgres' 65,535 bind parameters per statement, so its post failed with a 500 until the range was given up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f4ded062df
commit
953bbe9e39
3 files changed
+31
-7
No files matched your search
@@ -36,6 +36,9 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
|
||||
return stored ? unpackSegments(stored) : []
|
||||
}
|
||||
|
||||
/** Rows per INSERT, well below Postgres' 65,535 bind parameters. */
|
||||
export const INSERT_ROWS = 2000
|
||||
|
||||
class FileStore {
|
||||
/**
|
||||
* Inserts files, skipping those whose `file_key` exists. Returns the rows
|
||||
@@ -66,8 +69,13 @@ class FileStore {
|
||||
return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
|
||||
})
|
||||
if (rows.length === 0) return []
|
||||
const inserted = await client.knexQuery().table('files').insert(rows).onConflict().ignore().returning(['id', 'filename'])
|
||||
return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
|
||||
// A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
|
||||
const inserted: { id: number; filename: string }[] = []
|
||||
for (let i = 0; i < rows.length; i += INSERT_ROWS) {
|
||||
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + INSERT_ROWS)).onConflict().ignore().returning(['id', 'filename'])
|
||||
for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename })
|
||||
}
|
||||
return inserted
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user