Seed batches: insert files and binaries in chunks of 2,000 rows
A batch of a dense range (over ~5,400 files) exceeded Postgres' 65,535 bind parameters per statement, so its post failed with a 500 until the range was given up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f4ded062df
commit
953bbe9e39
3 files changed
+31
-7
No files matched your search
@@ -1,6 +1,6 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import FileStore from '#services/FileStore'
|
||||
import FileStore, { INSERT_ROWS } from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import QueueService from '#services/QueueService'
|
||||
@@ -78,10 +78,12 @@ class SeedService {
|
||||
file_id: fileIds.get(file.fileKey) ?? null,
|
||||
}))
|
||||
binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
await knex('binaries')
|
||||
.insert(binaries)
|
||||
.onConflict('file_key')
|
||||
.merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
|
||||
for (let i = 0; i < binaries.length; i += INSERT_ROWS) {
|
||||
await knex('binaries')
|
||||
.insert(binaries.slice(i, i + INSERT_ROWS))
|
||||
.onConflict('file_key')
|
||||
.merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
|
||||
}
|
||||
|
||||
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
|
||||
const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
|
||||
|
||||
Reference in new issue
Block a user