diff --git a/app/services/FileStore.ts b/app/services/FileStore.ts index 74538d2..b9bdeb8 100644 --- a/app/services/FileStore.ts +++ b/app/services/FileStore.ts @@ -36,8 +36,8 @@ export function loadedSegments(stored: Buffer | null): Segment[] { return stored ? unpackSegments(stored) : [] } -/** Rows per INSERT, well below Postgres' 65,535 bind parameters. */ -export const INSERT_ROWS = 2000 +/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */ +export const insertLimits = { rows: 2000 } class FileStore { /** @@ -71,8 +71,8 @@ class FileStore { if (rows.length === 0) return [] // A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns. const inserted: { id: number; filename: string }[] = [] - for (let i = 0; i < rows.length; i += INSERT_ROWS) { - const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + INSERT_ROWS)).onConflict().ignore().returning(['id', 'filename']) + for (let i = 0; i < rows.length; i += insertLimits.rows) { + const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename']) for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename }) } return inserted diff --git a/app/services/SeedService.ts b/app/services/SeedService.ts index 8240060..38c6a57 100644 --- a/app/services/SeedService.ts +++ b/app/services/SeedService.ts @@ -1,6 +1,6 @@ import db from '@adonisjs/lucid/services/db' import IngestService from '#services/IngestService' -import FileStore, { INSERT_ROWS } from '#services/FileStore' +import FileStore, { insertLimits } from '#services/FileStore' import SfvService from '#services/SfvService' import ContentService from '#services/ContentService' import QueueService from '#services/QueueService' @@ -78,9 +78,9 @@ class SeedService { file_id: fileIds.get(file.fileKey) ?? null, })) binaries.sort((a, b) => a.file_key.localeCompare(b.file_key)) - for (let i = 0; i < binaries.length; i += INSERT_ROWS) { + for (let i = 0; i < binaries.length; i += insertLimits.rows) { await knex('binaries') - .insert(binaries.slice(i, i + INSERT_ROWS)) + .insert(binaries.slice(i, i + insertLimits.rows)) .onConflict('file_key') .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any }) } diff --git a/tests/functional/seed.spec.ts b/tests/functional/seed.spec.ts index adfeeb4..934fb6d 100644 --- a/tests/functional/seed.spec.ts +++ b/tests/functional/seed.spec.ts @@ -6,7 +6,7 @@ import SeedService from '#services/SeedService' import QueueService from '#services/QueueService' import { FileAggregator } from '#lib/aggregate/index' import { toFragment, toSeedFile } from '#lib/seed/wire' -import { segmentsOf } from '#services/FileStore' +import { insertLimits, segmentsOf } from '#services/FileStore' function header(subject: string, n: number) { return { @@ -46,19 +46,23 @@ test.group('SeedService', (group) => { assert.include(again, { files: 0, existing: 2 }) }) - test('stores batches of more files than one INSERT can take', async ({ assert }) => { - // 12 columns per row: over 5,461 rows exceed Postgres' 65,535 bind parameters. - const aggregator = new FileAggregator() - const segments = Array.from({ length: 6000 }, (_, i) => - segmentFromHeader(header(`Dense.Upload - [${i + 1}/6000] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)! - ) - const files = aggregator.add(segments).map(toSeedFile) - assert.lengthOf(files, 6000) - const result = await SeedService.store(files) - assert.include(result, { files: 6000 }) - const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count') - assert.equal(Number(count), 6000) - }).timeout(60_000) + test('stores a batch in several INSERTs', async ({ assert }) => { + // Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns. + const rows = insertLimits.rows + insertLimits.rows = 50 + try { + const aggregator = new FileAggregator() + const segments = Array.from({ length: 120 }, (_, i) => + segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)! + ) + const files = aggregator.add(segments).map(toSeedFile) + assert.include(await SeedService.store(files), { files: 120 }) + const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count') + assert.equal(Number(count), 120) + } finally { + insertLimits.rows = rows + } + }) test('fragments are kept near the range edges only', async ({ assert }) => { const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`