Seed batches: test chunked inserts with a lowered chunk size

The 6,000-file test took 29 s and failed CI; 120 files in chunks of 50
cover the same path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/FileStore.ts

@@ -36,8 +36,8 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
   return stored ? unpackSegments(stored) : []
 }
 
-/** Rows per INSERT, well below Postgres' 65,535 bind parameters. */
-export const INSERT_ROWS = 2000
+/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
+export const insertLimits = { rows: 2000 }
 
 class FileStore {
   /**
@@ -71,8 +71,8 @@ class FileStore {
     if (rows.length === 0) return []
     // A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
     const inserted: { id: number; filename: string }[] = []
-    for (let i = 0; i < rows.length; i += INSERT_ROWS) {
-      const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + INSERT_ROWS)).onConflict().ignore().returning(['id', 'filename'])
+    for (let i = 0; i < rows.length; i += insertLimits.rows) {
+      const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename'])
       for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename })
     }
     return inserted

app/services/SeedService.ts

@@ -1,6 +1,6 @@
 import db from '@adonisjs/lucid/services/db'
 import IngestService from '#services/IngestService'
-import FileStore, { INSERT_ROWS } from '#services/FileStore'
+import FileStore, { insertLimits } from '#services/FileStore'
 import SfvService from '#services/SfvService'
 import ContentService from '#services/ContentService'
 import QueueService from '#services/QueueService'
@@ -78,9 +78,9 @@ class SeedService {
       file_id: fileIds.get(file.fileKey) ?? null,
     }))
     binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
-    for (let i = 0; i < binaries.length; i += INSERT_ROWS) {
+    for (let i = 0; i < binaries.length; i += insertLimits.rows) {
       await knex('binaries')
-        .insert(binaries.slice(i, i + INSERT_ROWS))
+        .insert(binaries.slice(i, i + insertLimits.rows))
         .onConflict('file_key')
         .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
     }

tests/functional/seed.spec.ts

@@ -6,7 +6,7 @@ import SeedService from '#services/SeedService'
 import QueueService from '#services/QueueService'
 import { FileAggregator } from '#lib/aggregate/index'
 import { toFragment, toSeedFile } from '#lib/seed/wire'
-import { segmentsOf } from '#services/FileStore'
+import { insertLimits, segmentsOf } from '#services/FileStore'
 
 function header(subject: string, n: number) {
   return {
@@ -46,19 +46,23 @@ test.group('SeedService', (group) => {
     assert.include(again, { files: 0, existing: 2 })
   })
 
-  test('stores batches of more files than one INSERT can take', async ({ assert }) => {
-    // 12 columns per row: over 5,461 rows exceed Postgres' 65,535 bind parameters.
-    const aggregator = new FileAggregator<IngestSegment>()
-    const segments = Array.from({ length: 6000 }, (_, i) =>
-      segmentFromHeader(header(`Dense.Upload - [${i + 1}/6000] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
-    )
-    const files = aggregator.add(segments).map(toSeedFile)
-    assert.lengthOf(files, 6000)
-    const result = await SeedService.store(files)
-    assert.include(result, { files: 6000 })
-    const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
-    assert.equal(Number(count), 6000)
-  }).timeout(60_000)
+  test('stores a batch in several INSERTs', async ({ assert }) => {
+    // Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
+    const rows = insertLimits.rows
+    insertLimits.rows = 50
+    try {
+      const aggregator = new FileAggregator<IngestSegment>()
+      const segments = Array.from({ length: 120 }, (_, i) =>
+        segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
+      )
+      const files = aggregator.add(segments).map(toSeedFile)
+      assert.include(await SeedService.store(files), { files: 120 })
+      const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
+      assert.equal(Number(count), 120)
+    } finally {
+      insertLimits.rows = rows
+    }
+  })
 
   test('fragments are kept near the range edges only', async ({ assert }) => {
     const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`