Keep binaries until inspected; seeding writes binaries only for inspection
Test and deploy / test (push) Successful in 1m24s
Test and deploy / deploy (push) Successful in 6m43s

Par2 naming needs the fingerprint inspection stores on a binary, and the
GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before
the inspection queue reached them. binaries.awaiting_inspection protects
them for up to 7 days; the GC deletes use partial indexes so they don't
read past waiting rows. Seeding no longer creates binaries for files that
aren't inspected (~85% of a.b.ath), which only churned the table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 20:03:43 +02:00
1 parent cc861ce8ae
commit b2cc75fdd0
11 files changed
+149 -35

No files matched your search

+15 -4
View File
@@ -37,15 +37,26 @@ test.group('SeedService', (group) => {
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
const release = await db.from('releases').where('id', stored.release_id).first()
assert.equal(release.name, segments[0].release!.name) // named as live indexing would
const binary = await db.from('binaries').where('file_key', stored.file_key).first()
assert.equal(Number(binary.file_id), Number(stored.id))
assert.isNotNull(binary.completed_at)
// A readable name isn't inspected: no binary (LOG.md 2026-10-06 (11)).
assert.isNull(await db.from('binaries').where('file_key', stored.file_key).first())
// The same files again (a crosspost, a second run): nothing new.
const again = await SeedService.store(files)
assert.include(again, { files: 0, existing: 2 })
})
test('an obfuscated file keeps a binary until inspection has fingerprinted it', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const subject = 'Xq8r2KdLm3Zw91PqA7zz [1/1] - "Xq8r2KdLm3Zw91PqA7zz.bin" yEnc (%d/2)'
const segments = [1, 2].map((n) => segmentFromHeader(header(subject.replace('%d', String(n)), n), 'alt.binaries.test', n)!)
const files = aggregator.add(segments).map(toSeedFile)
const result = await SeedService.store(files)
assert.include(result, { files: 1, inspections: 1 })
const binary = await db.from('binaries').where('file_key', files[0].fileKey).first()
assert.isTrue(binary.awaiting_inspection)
assert.isNotNull(binary.file_id)
})
test('stores a batch in several INSERTs', async ({ assert }) => {
// Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
const rows = insertLimits.rows
@@ -57,7 +68,7 @@ test.group('SeedService', (group) => {
)
const files = aggregator.add(segments).map(toSeedFile)
assert.include(await SeedService.store(files), { files: 120 })
const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
const [{ count }] = await db.from('files').whereLike('filename', 'dense.%').count('* as count')
assert.equal(Number(count), 120)
} finally {
insertLimits.rows = rows