Idle connections fetch capture ranges into the corpus
The spool only fetches what workers lease soon, in priority order, and the tail never ends: after a reset the 1,400 capture ranges waited behind it while 34 of 40 connections idled. Up to CORPUS_AHEAD (8) planned capture ranges whose chunks aren't all stored are now targets after everything else, fetched into the corpus only (no spool); a later lease reads them from disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0fafe71b21
commit
7b436f3f77
5 files changed
+106
-3
No files matched your search
@@ -0,0 +1,37 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { gzipSync } from 'node:zlib'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import Corpus from '#services/corpus'
|
||||
|
||||
test.group('Corpus', (group) => {
|
||||
let dir: string
|
||||
let before: string | undefined
|
||||
group.each.setup(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'corpus-'))
|
||||
before = coordinatorConfig.corpus.dir
|
||||
coordinatorConfig.corpus.dir = dir
|
||||
})
|
||||
group.each.teardown(async () => {
|
||||
coordinatorConfig.corpus.dir = before
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test('knows the chunks it holds, and a range once all are there', async ({ assert }) => {
|
||||
const range = { group: 'alt.binaries.test', start: 1, end: 200_000 }
|
||||
const first = { from: 1, to: 100_000 }
|
||||
const second = { from: 100_001, to: 200_000 }
|
||||
assert.isFalse(await Corpus.has(range, first))
|
||||
await Corpus.write(range, first, gzipSync('1\tsubject\n'), 2)
|
||||
assert.isTrue(await Corpus.has(range, first))
|
||||
assert.isFalse(await Corpus.complete(range))
|
||||
// A reply cut short isn't kept.
|
||||
await Corpus.write(range, second, Buffer.from('not gzip'), 2)
|
||||
assert.isFalse(await Corpus.has(range, second))
|
||||
await Corpus.write(range, second, gzipSync('2\tsubject\n'), 2)
|
||||
assert.isTrue(await Corpus.complete(range))
|
||||
assert.equal((await Corpus.read(range, second))?.length, gzipSync('2\tsubject\n').length)
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user