Daan MeijerandClaude Opus 5.5
7b436f3f77
The spool only fetches what workers lease soon, in priority order, and the tail never ends: after a reset the 1,400 capture ranges waited behind it while 34 of 40 connections idled. Up to CORPUS_AHEAD (8) planned capture ranges whose chunks aren't all stored are now targets after everything else, fetched into the corpus only (no spool); a later lease reads them from disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
38 lines
1.5 KiB
TypeScript
38 lines
1.5 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { mkdtemp, rm } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { gzipSync } from 'node:zlib'
|
|
import coordinatorConfig from '#config/coordinator'
|
|
import Corpus from '#services/corpus'
|
|
|
|
test.group('Corpus', (group) => {
|
|
let dir: string
|
|
let before: string | undefined
|
|
group.each.setup(async () => {
|
|
dir = await mkdtemp(join(tmpdir(), 'corpus-'))
|
|
before = coordinatorConfig.corpus.dir
|
|
coordinatorConfig.corpus.dir = dir
|
|
})
|
|
group.each.teardown(async () => {
|
|
coordinatorConfig.corpus.dir = before
|
|
await rm(dir, { recursive: true, force: true })
|
|
})
|
|
|
|
test('knows the chunks it holds, and a range once all are there', async ({ assert }) => {
|
|
const range = { group: 'alt.binaries.test', start: 1, end: 200_000 }
|
|
const first = { from: 1, to: 100_000 }
|
|
const second = { from: 100_001, to: 200_000 }
|
|
assert.isFalse(await Corpus.has(range, first))
|
|
await Corpus.write(range, first, gzipSync('1\tsubject\n'), 2)
|
|
assert.isTrue(await Corpus.has(range, first))
|
|
assert.isFalse(await Corpus.complete(range))
|
|
// A reply cut short isn't kept.
|
|
await Corpus.write(range, second, Buffer.from('not gzip'), 2)
|
|
assert.isFalse(await Corpus.has(range, second))
|
|
await Corpus.write(range, second, gzipSync('2\tsubject\n'), 2)
|
|
assert.isTrue(await Corpus.complete(range))
|
|
assert.equal((await Corpus.read(range, second))?.length, gzipSync('2\tsubject\n').length)
|
|
})
|
|
})
|