From 953bbe9e3912d39347bf9de2c2f70932188078cf Mon Sep 17 00:00:00 2001
From: Daan Meijer <daan@studioseptember.nl>
Date: Tue, 6 Oct 2026 00:01:32 +0200
Subject: [PATCH] Seed batches: insert files and binaries in chunks of 2,000
 rows

A batch of a dense range (over ~5,400 files) exceeded Postgres' 65,535
bind parameters per statement, so its post failed with a 500 until the
range was given up.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 app/services/FileStore.ts     | 12 ++++++++++--
 app/services/SeedService.ts   | 12 +++++++-----
 tests/functional/seed.spec.ts | 14 ++++++++++++++
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/app/services/FileStore.ts b/app/services/FileStore.ts
index 1d83a38..74538d2 100644
--- a/app/services/FileStore.ts
+++ b/app/services/FileStore.ts
@@ -36,6 +36,9 @@ export function loadedSegments(stored: Buffer | null): Segment[] {
   return stored ? unpackSegments(stored) : []
 }
 
+/** Rows per INSERT, well below Postgres' 65,535 bind parameters. */
+export const INSERT_ROWS = 2000
+
 class FileStore {
   /**
    * Inserts files, skipping those whose `file_key` exists. Returns the rows
@@ -66,8 +69,13 @@ class FileStore {
         return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
       })
     if (rows.length === 0) return []
-    const inserted = await client.knexQuery().table('files').insert(rows).onConflict().ignore().returning(['id', 'filename'])
-    return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
+    // A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
+    const inserted: { id: number; filename: string }[] = []
+    for (let i = 0; i < rows.length; i += INSERT_ROWS) {
+      const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + INSERT_ROWS)).onConflict().ignore().returning(['id', 'filename'])
+      for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename })
+    }
+    return inserted
   }
 }
 
diff --git a/app/services/SeedService.ts b/app/services/SeedService.ts
index 0a1668a..8240060 100644
--- a/app/services/SeedService.ts
+++ b/app/services/SeedService.ts
@@ -1,6 +1,6 @@
 import db from '@adonisjs/lucid/services/db'
 import IngestService from '#services/IngestService'
-import FileStore from '#services/FileStore'
+import FileStore, { INSERT_ROWS } from '#services/FileStore'
 import SfvService from '#services/SfvService'
 import ContentService from '#services/ContentService'
 import QueueService from '#services/QueueService'
@@ -78,10 +78,12 @@ class SeedService {
       file_id: fileIds.get(file.fileKey) ?? null,
     }))
     binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
-    await knex('binaries')
-      .insert(binaries)
-      .onConflict('file_key')
-      .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
+    for (let i = 0; i < binaries.length; i += INSERT_ROWS) {
+      await knex('binaries')
+        .insert(binaries.slice(i, i + INSERT_ROWS))
+        .onConflict('file_key')
+        .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })
+    }
 
     // As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
     const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
diff --git a/tests/functional/seed.spec.ts b/tests/functional/seed.spec.ts
index 25d7105..adfeeb4 100644
--- a/tests/functional/seed.spec.ts
+++ b/tests/functional/seed.spec.ts
@@ -46,6 +46,20 @@ test.group('SeedService', (group) => {
     assert.include(again, { files: 0, existing: 2 })
   })
 
+  test('stores batches of more files than one INSERT can take', async ({ assert }) => {
+    // 12 columns per row: over 5,461 rows exceed Postgres' 65,535 bind parameters.
+    const aggregator = new FileAggregator<IngestSegment>()
+    const segments = Array.from({ length: 6000 }, (_, i) =>
+      segmentFromHeader(header(`Dense.Upload - [${i + 1}/6000] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
+    )
+    const files = aggregator.add(segments).map(toSeedFile)
+    assert.lengthOf(files, 6000)
+    const result = await SeedService.store(files)
+    assert.include(result, { files: 6000 })
+    const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
+    assert.equal(Number(count), 6000)
+  }).timeout(60_000)
+
   test('fragments are kept near the range edges only', async ({ assert }) => {
     const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
     const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!