Daan MeijerandClaude Opus 5.5
0613984de9
A poster re-uploading under the same name made one release of all uploads: in a sample, 24% of releases with 2+ files had a file name more than once (4.3% of files extra copies), 70% of those copies posted a day or more apart; their NZBs listed each copy. - File assembly keeps a re-upload's copy apart: a part already in every open copy, posted more than 3 h from them, starts another copy; a missing part fills the nearest copy (a re-send of what was missing), a re-sent part near a copy that has it is ignored as before. - A key's files in a batch split into uploads where posting pauses for more than 3 h, counted from the latest article posted so far (a file that takes hours to post doesn't split it); each upload is judged and chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry their upload). - The catalog matches a chunk to the release of its key whose articles the range's group owns and whose posting window is within 3 h, else starts one (key, key:2, …): releases get identity_key, group_id, last_posted_at. Keys are locked in order (advisory locks), so concurrent batches neither deadlock nor both start one; the merger writes in key order too. The same upload multi-posted to other groups is a release per group. - The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies come later (releases:upload-gaps). releases:duplicates measures the duplicate names. Both implementations; check.ts compares the windows and uploads too (identical on all captures; Rust tests mirror the TypeScript cases). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
56 lines
2.8 KiB
TypeScript
56 lines
2.8 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { FileAggregator, type AggregatableSegment } from '#lib/files/index'
|
|
|
|
const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
|
|
fileKey,
|
|
number,
|
|
totalParts,
|
|
messageId,
|
|
bytes: 100,
|
|
})
|
|
|
|
test.group('aggregate / FileAggregator', () => {
|
|
test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
|
|
assert.equal(aggregator.openFiles, 1)
|
|
const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
|
|
assert.equal(done.fileKey, 'a')
|
|
assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
|
|
test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
|
|
assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
|
|
assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
|
|
})
|
|
|
|
test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
|
|
const aggregator = new FileAggregator()
|
|
aggregator.add([seg('old', 1, 3)])
|
|
aggregator.add([seg('new', 1, 3)])
|
|
aggregator.add([seg('new', 2, 3)])
|
|
assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
|
|
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
test('a re-upload is a copy of its own; a missing part sent later fills the first', ({ assert }) => {
|
|
const hour = 3_600_000
|
|
const at = (number: number, h: number, id: string) => ({ ...seg('a', number, 3, `${id}@test`), postedAt: h * hour })
|
|
const aggregator = new FileAggregator()
|
|
// Parts 1 and 2 at 10:00; part 1 again right after (a re-send: ignored).
|
|
assert.lengthOf(aggregator.add([at(1, 10, 'a1'), at(2, 10, 'a2'), at(1, 10.1, 'resend')]), 0)
|
|
// A day later: parts 1 and 2 again (a re-upload, another copy), and part 3 (fills the nearest copy lacking it: the new one).
|
|
assert.lengthOf(aggregator.add([at(1, 34, 'b1'), at(2, 34, 'b2')]), 0)
|
|
assert.equal(aggregator.openFiles, 2)
|
|
const [second] = aggregator.add([at(3, 34.2, 'b3')])
|
|
assert.deepEqual(second.segments.map((s) => s.messageId), ['b1@test', 'b2@test', 'b3@test'])
|
|
// The first copy's missing part, sent two days later: it completes the first.
|
|
const [first] = aggregator.add([at(3, 58, 'a3')])
|
|
assert.deepEqual(first.segments.map((s) => s.messageId), ['a1@test', 'a2@test', 'a3@test'])
|
|
assert.equal(aggregator.openFiles, 0)
|
|
})
|
|
})
|