Seed fragments: keep only those near the range edges, unstaged
Test and deploy / test (push) Successful in 1m5s
Test and deploy / deploy (push) Successful in 1m48s

About 5% of seeded headers are fragments. They went through staging,
where they waited 30 minutes for parts that were never coming (the
neighbouring range is worked on by another node, often hours later) and
expired: the staging table grew to 22 GB and the NVMe part of the database
reached its 60 GB limit, which stopped leasing and the backfill. Files cut
by a range edge can still be completed by the neighbouring range, so
fragments within 50k articles of an edge go straight into binaries and
parts; the rest, files with articles missing for good, are dropped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-05 20:30:24 +02:00
1 parent 530a752cca
commit 8d86f7d489
5 files changed
+35 -15

No files matched your search

+11 -1
View File
@@ -5,7 +5,7 @@ import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
import SeedService from '#services/SeedService'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toSeedFile } from '#lib/seed/wire'
import { toFragment, toSeedFile } from '#lib/seed/wire'
import { segmentsOf } from '#services/FileStore'
function header(subject: string, n: number) {
@@ -45,4 +45,14 @@ test.group('SeedService', (group) => {
const again = await SeedService.store(files)
assert.include(again, { files: 0, existing: 2 })
})
test('fragments are kept near the range edges only', async ({ assert }) => {
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
const middle = segmentFromHeader(header(subject('middle.bin'), 2), 'alt.binaries.test', 1_500_000)!
const kept = await SeedService.storeFragments([toFragment(edge), toFragment(middle)], { start: 1_000_000, end: 2_999_999 })
assert.equal(kept, 1)
assert.isOk(await db.from('binaries').where('name', 'edge.bin').first())
assert.isNotOk(await db.from('binaries').where('name', 'middle.bin').first())
})
})
+2 -3
View File
@@ -48,9 +48,8 @@ test.group('Seed API', (group) => {
assert.include(((await posted.json()) as any).stored, { files: 1 })
const file = await db.from('files').where('filename', 'Api.Test.2026.mkv').first()
assert.equal(file.parts, 2)
// The lone .nfo segment took the regular path (staged or stored as a binary).
const fragment = (await db.from('staged_segments').count('* as n').first()).n + (await db.from('binaries').where('name', 'Api.Test.2026.nfo').count('* as n').first()).n
assert.isAbove(Number(fragment), 0)
// The lone .nfo segment sits near the range's edge: kept as a binary (not staged) for the neighbouring range.
assert.isOk(await db.from('binaries').where('name', 'Api.Test.2026.nfo').first())
const row = await db.from('seed_ranges').where('id', range.id).first()
assert.equal(Number(row.next_id), 51)