Stop refilling parts of binaries that are already a file
Crossposts, reposts and backfill ranges keep bringing articles of files that worker:file already created. Ingest inserted their header parts again, and GC's copied-parts pass (which searched all binaries for parts on every batch) removed only 150k-450k per sweep: parts grew to 68M rows, 56M of them belonging to finished files, and the 23 GB table paused the backfill. - Ingest skips header parts (no per-part CRC) for binaries with a file_id. - GC drops copied parts with a cursor over binaries, 2000 per statement. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d3e774a489
commit
2178847513
3 files changed
+60
-14
No files matched your search
@@ -4,6 +4,7 @@ import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import IngestService, { type IngestSegment } from '#services/IngestService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { fileKey } from '#lib/subject/index'
|
||||
import FileStore from '#services/FileStore'
|
||||
|
||||
function segment(overrides: Partial<IngestSegment>): IngestSegment {
|
||||
const poster = 'ingest-test@example.invalid'
|
||||
@@ -51,6 +52,25 @@ test.group('IngestService', (group) => {
|
||||
assert.isAbove(+after.last_seen_at, Date.now() - 60_000)
|
||||
})
|
||||
|
||||
test('a binary that is a file already takes no new header parts', async ({ assert }) => {
|
||||
await IngestService.ingest([segment({ number: 1 })])
|
||||
const binary = await binaryFor(segment({}).fileKey)
|
||||
const now = new Date()
|
||||
const [file] = await FileStore.insert([
|
||||
{ filename: 'test-file.bin', poster: 'x', date: now.getTime(), parts: 3, size: 300, groups: '[]', segments: [], created_at: now, updated_at: now },
|
||||
])
|
||||
await db.from('binaries').where('id', binary.id).update({ file_id: file.id })
|
||||
await db.from('parts').where('binary_id', binary.id).delete()
|
||||
|
||||
// The same articles again (a crosspost, a backfill range): nothing to store.
|
||||
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test' }), segment({ number: 3, messageId: 'part3@test' })])
|
||||
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 0)
|
||||
|
||||
// Body-derived parts (with a per-part CRC) are still kept.
|
||||
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', pcrc32: 'abcd1234' })])
|
||||
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 1)
|
||||
})
|
||||
|
||||
test('ingesting the same segments twice is a no-op', async ({ assert }) => {
|
||||
const segments = [segment({ number: 1 }), segment({ number: 2, messageId: 'part2@test' })]
|
||||
await IngestService.ingest(segments)
|
||||
|
||||
Reference in new issue
Block a user