Rebuild: re-uploads are releases of their own
A poster re-uploading under the same name made one release of all uploads: in a sample, 24% of releases with 2+ files had a file name more than once (4.3% of files extra copies), 70% of those copies posted a day or more apart; their NZBs listed each copy. - File assembly keeps a re-upload's copy apart: a part already in every open copy, posted more than 3 h from them, starts another copy; a missing part fills the nearest copy (a re-send of what was missing), a re-sent part near a copy that has it is ignored as before. - A key's files in a batch split into uploads where posting pauses for more than 3 h, counted from the latest article posted so far (a file that takes hours to post doesn't split it); each upload is judged and chunked on its own (ReleaseChunk.upload, .postedLast; candidates carry their upload). - The catalog matches a chunk to the release of its key whose articles the range's group owns and whose posting window is within 3 h, else starts one (key, key:2, …): releases get identity_key, group_id, last_posted_at. Keys are locked in order (advisory locks), so concurrent batches neither deadlock nor both start one; the merger writes in key order too. The same upload multi-posted to other groups is a release per group. - The gap: 91% of uploads post within 3 h; 73% of re-uploaded copies come later (releases:upload-gaps). releases:duplicates measures the duplicate names. Both implementations; check.ts compares the windows and uploads too (identical on all captures; Rust tests mirror the TypeScript cases). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d0f3497ec9
commit
0613984de9
15 files changed
+715
-140
No files matched your search
@@ -81,4 +81,20 @@ test.group('edge / taking over a range', () => {
|
||||
indexer.add(post(500_001, 'A.Release.2020.mkv', 1), 600_000)
|
||||
assert.deepEqual([indexer.take().from, indexer.take().from], [500_001, 600_000])
|
||||
})
|
||||
test('uploads split where posting pauses for more than 3 hours, counted from the latest article', ({ assert }) => {
|
||||
const art = (n: number, subject: string, minutes: number): Overview => [n, { 'subject': subject, 'from': 'p <p@x>', 'date': new Date(Date.UTC(2026, 9, 5, 10) + minutes * 60_000).toUTCString(), 'message-id': `<u${n}@test>`, ':bytes': '1000', 'xref': `news.example ${GROUP}:${n}` }]
|
||||
const ix = new RangeIndexer({ group: GROUP, start: 1, end: 10_000, rules })
|
||||
ix.add([
|
||||
// A big file posted over 4 hours, then its par2 right after: one upload.
|
||||
...Array.from({ length: 5 }, (_, i) => art(100 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, i * 60)),
|
||||
art(200, 'Big.Release [2/2] - "Big.Release.par2" yEnc (1/1)', 245),
|
||||
// The same file again a day later: another.
|
||||
...Array.from({ length: 5 }, (_, i) => art(300 + i, `Big.Release [1/2] - "Big.Release.mkv" yEnc (${i + 1}/5)`, 1440 + i)),
|
||||
], 10_001)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
assert.deepEqual(batch.releases.map((r) => [r.upload, r.files]), [[0, 2], [1, 1]])
|
||||
assert.equal(batch.releases[0].postedLast! - batch.releases[0].postedAt!, 245 * 60_000)
|
||||
assert.equal(batch.releases[0].key, batch.releases[1].key)
|
||||
})
|
||||
})
|
||||
@@ -36,4 +36,20 @@ test.group('aggregate / FileAggregator', () => {
|
||||
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
|
||||
assert.equal(aggregator.openFiles, 0)
|
||||
})
|
||||
test('a re-upload is a copy of its own; a missing part sent later fills the first', ({ assert }) => {
|
||||
const hour = 3_600_000
|
||||
const at = (number: number, h: number, id: string) => ({ ...seg('a', number, 3, `${id}@test`), postedAt: h * hour })
|
||||
const aggregator = new FileAggregator()
|
||||
// Parts 1 and 2 at 10:00; part 1 again right after (a re-send: ignored).
|
||||
assert.lengthOf(aggregator.add([at(1, 10, 'a1'), at(2, 10, 'a2'), at(1, 10.1, 'resend')]), 0)
|
||||
// A day later: parts 1 and 2 again (a re-upload, another copy), and part 3 (fills the nearest copy lacking it: the new one).
|
||||
assert.lengthOf(aggregator.add([at(1, 34, 'b1'), at(2, 34, 'b2')]), 0)
|
||||
assert.equal(aggregator.openFiles, 2)
|
||||
const [second] = aggregator.add([at(3, 34.2, 'b3')])
|
||||
assert.deepEqual(second.segments.map((s) => s.messageId), ['b1@test', 'b2@test', 'b3@test'])
|
||||
// The first copy's missing part, sent two days later: it completes the first.
|
||||
const [first] = aggregator.add([at(3, 58, 'a3')])
|
||||
assert.deepEqual(first.segments.map((s) => s.messageId), ['a1@test', 'a2@test', 'a3@test'])
|
||||
assert.equal(aggregator.openFiles, 0)
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user