Keep binaries until inspected; seeding writes binaries only for inspection
Par2 naming needs the fingerprint inspection stores on a binary, and the GC deleted seeded binaries (6 h unfingerprinted, 48 h stale) long before the inspection queue reached them. binaries.awaiting_inspection protects them for up to 7 days; the GC deletes use partial indexes so they don't read past waiting rows. Seeding no longer creates binaries for files that aren't inspected (~85% of a.b.ath), which only churned the table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cc861ce8ae
commit
b2cc75fdd0
11 files changed
+149
-35
No files matched your search
@@ -37,15 +37,26 @@ test.group('SeedService', (group) => {
|
||||
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
|
||||
const release = await db.from('releases').where('id', stored.release_id).first()
|
||||
assert.equal(release.name, segments[0].release!.name) // named as live indexing would
|
||||
const binary = await db.from('binaries').where('file_key', stored.file_key).first()
|
||||
assert.equal(Number(binary.file_id), Number(stored.id))
|
||||
assert.isNotNull(binary.completed_at)
|
||||
// A readable name isn't inspected: no binary (LOG.md 2026-10-06 (11)).
|
||||
assert.isNull(await db.from('binaries').where('file_key', stored.file_key).first())
|
||||
|
||||
// The same files again (a crosspost, a second run): nothing new.
|
||||
const again = await SeedService.store(files)
|
||||
assert.include(again, { files: 0, existing: 2 })
|
||||
})
|
||||
|
||||
test('an obfuscated file keeps a binary until inspection has fingerprinted it', async ({ assert }) => {
|
||||
const aggregator = new FileAggregator<IngestSegment>()
|
||||
const subject = 'Xq8r2KdLm3Zw91PqA7zz [1/1] - "Xq8r2KdLm3Zw91PqA7zz.bin" yEnc (%d/2)'
|
||||
const segments = [1, 2].map((n) => segmentFromHeader(header(subject.replace('%d', String(n)), n), 'alt.binaries.test', n)!)
|
||||
const files = aggregator.add(segments).map(toSeedFile)
|
||||
const result = await SeedService.store(files)
|
||||
assert.include(result, { files: 1, inspections: 1 })
|
||||
const binary = await db.from('binaries').where('file_key', files[0].fileKey).first()
|
||||
assert.isTrue(binary.awaiting_inspection)
|
||||
assert.isNotNull(binary.file_id)
|
||||
})
|
||||
|
||||
test('stores a batch in several INSERTs', async ({ assert }) => {
|
||||
// Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
|
||||
const rows = insertLimits.rows
|
||||
@@ -57,7 +68,7 @@ test.group('SeedService', (group) => {
|
||||
)
|
||||
const files = aggregator.add(segments).map(toSeedFile)
|
||||
assert.include(await SeedService.store(files), { files: 120 })
|
||||
const [{ count }] = await db.from('binaries').whereLike('name', 'dense.%').whereNotNull('file_id').count('* as count')
|
||||
const [{ count }] = await db.from('files').whereLike('filename', 'dense.%').count('* as count')
|
||||
assert.equal(Number(count), 120)
|
||||
} finally {
|
||||
insertLimits.rows = rows
|
||||
|
||||
Reference in new issue
Block a user