Stop refilling parts of binaries that are already a file
Test and deploy / test (push) Successful in 52s
Test and deploy / deploy (push) Successful in 1m23s

Crossposts, reposts and backfill ranges keep bringing articles of files
that worker:file already created. Ingest inserted their header parts again,
and GC's copied-parts pass (which searched all binaries for parts on every
batch) removed only 150k-450k per sweep: parts grew to 68M rows, 56M of
them belonging to finished files, and the 23 GB table paused the backfill.

- Ingest skips header parts (no per-part CRC) for binaries with a file_id.
- GC drops copied parts with a cursor over binaries, 2000 per statement.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-04 02:53:11 +02:00
1 parent d3e774a489
commit 2178847513
3 files changed
+60 -14

No files matched your search

+20
View File
@@ -4,6 +4,7 @@ import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import QueueService from '#services/QueueService'
import { fileKey } from '#lib/subject/index'
import FileStore from '#services/FileStore'
function segment(overrides: Partial<IngestSegment>): IngestSegment {
const poster = 'ingest-test@example.invalid'
@@ -51,6 +52,25 @@ test.group('IngestService', (group) => {
assert.isAbove(+after.last_seen_at, Date.now() - 60_000)
})
test('a binary that is a file already takes no new header parts', async ({ assert }) => {
await IngestService.ingest([segment({ number: 1 })])
const binary = await binaryFor(segment({}).fileKey)
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'test-file.bin', poster: 'x', date: now.getTime(), parts: 3, size: 300, groups: '[]', segments: [], created_at: now, updated_at: now },
])
await db.from('binaries').where('id', binary.id).update({ file_id: file.id })
await db.from('parts').where('binary_id', binary.id).delete()
// The same articles again (a crosspost, a backfill range): nothing to store.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test' }), segment({ number: 3, messageId: 'part3@test' })])
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 0)
// Body-derived parts (with a per-part CRC) are still kept.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', pcrc32: 'abcd1234' })])
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 1)
})
test('ingesting the same segments twice is a no-op', async ({ assert }) => {
const segments = [segment({ number: 1 }), segment({ number: 2, messageId: 'part2@test' })]
await IngestService.ingest(segments)