Daan MeijerandClaude Opus 5.5
530a752cca
- A range reported to its end but not finished (its node went away at the very end) was leased again with start past end; nodes asked for "from 49208000000 to 49207999999", got 400, and the range failed after five leases. The orchestrator now marks such ranges done before leasing, and runRange reports them done without fetching. - Some old ranges come so slowly that one XOVER of 20k headers takes over two minutes: five ranges failed on "NNTP reply timed out". The relay pool's timeout is SEED_NNTP_TIMEOUT_MS (default 10 minutes, the node's HTTP timeout). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
57 lines
2.3 KiB
TypeScript
57 lines
2.3 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { FileAggregator, type AggregatableSegment } from '#lib/aggregate/index'
|
|
|
|
const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
|
|
fileKey,
|
|
number,
|
|
totalParts,
|
|
messageId,
|
|
bytes: 100,
|
|
})
|
|
|
|
test.group('aggregate / FileAggregator', () => {
|
|
test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
|
|
assert.equal(aggregator.openFiles, 1)
|
|
const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
|
|
assert.equal(done.fileKey, 'a')
|
|
assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
|
|
test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
|
|
assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
|
|
assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
|
|
})
|
|
|
|
test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
aggregator.add([seg('old', 1, 3)])
|
|
aggregator.add([seg('new', 1, 3)])
|
|
aggregator.add([seg('new', 2, 3)])
|
|
assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
|
|
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
})
|
|
|
|
test.group('seed / runRange', () => {
|
|
test('a range already past its end only reports that it is done', async ({ assert }) => {
|
|
const { runRange } = await import('#lib/seed/runner')
|
|
const batches: any[] = []
|
|
await runRange({
|
|
client: { group: async () => assert.fail('no GROUP'), xover: async () => assert.fail('no XOVER') },
|
|
group: 'a.b.test',
|
|
start: 101,
|
|
end: 100,
|
|
sink: async (batch) => {
|
|
batches.push(batch)
|
|
},
|
|
})
|
|
assert.deepEqual(batches.map((b) => [b.next, b.headers, b.files.length]), [[101, 0, 0]])
|
|
})
|
|
})
|