Daan MeijerandClaude Opus 5.5
68eb9a44fd
A seeding node fetches a range over one NNTP connection, parses headers with the live rules, groups them into files in memory (lib/aggregate) and stores per complete file its release, the file and a completed binary (SeedService), instead of one parts row per article. What doesn't complete within the window goes through the regular ingest path as fragments. `--dry-run` only fetches and aggregates, to measure a node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
40 lines
1.7 KiB
TypeScript
40 lines
1.7 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { FileAggregator, type AggregatableSegment } from '#lib/aggregate/index'
|
|
|
|
const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
|
|
fileKey,
|
|
number,
|
|
totalParts,
|
|
messageId,
|
|
bytes: 100,
|
|
})
|
|
|
|
test.group('aggregate / FileAggregator', () => {
|
|
test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
|
|
assert.equal(aggregator.openFiles, 1)
|
|
const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
|
|
assert.equal(done.fileKey, 'a')
|
|
assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
|
|
test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
|
|
assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
|
|
assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
|
|
})
|
|
|
|
test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
aggregator.add([seg('old', 1, 3)])
|
|
aggregator.add([seg('new', 1, 3)])
|
|
aggregator.add([seg('new', 2, 3)])
|
|
assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
|
|
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
})
|