Rebuild: fragments only for files an edge cut; lone random articles are per-article
Measured on two neighbouring boneless ranges (merging their fragments): - 99.8% of fragment files were single articles of a random-named 'file' of >100 parts: a random subject per article, which never completes from headers. They now count as per-article. - A file is kept as a fragment when its first part seen is above 1 and comes within the start window (the idle window: uploads in busy groups spread over minutes, and a 50k-article edge zone is ~30 s of boneless), or when it was still open at the range's end with later parts missing. Anything else missing parts is incomplete and only counted. Per range: 70,308 → 130 fragment files; merged, 47 of the 48 files met complete (57,558 segments, ~8% of the range's kept segments). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5ba93f558c
commit
2c14315f94
3 files changed
+70
-25
No files matched your search
@@ -55,13 +55,22 @@ test.group('edge / RangeIndexer', () => {
|
||||
assert.equal(batch.next, 1_000_001)
|
||||
})
|
||||
|
||||
test('a file cut off near an edge leaves as fragments, one in the middle is only counted', ({ assert }) => {
|
||||
test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => {
|
||||
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
|
||||
indexer.add([...post(5, 'Edge.File.2020.part1.rar', 3).slice(0, 2), ...post(500_000, 'Middle.File.2020.part1.rar', 3).slice(0, 2)], 600_000)
|
||||
indexer.add([
|
||||
// Parts 2–3 right after the start: part 1 is in the previous range.
|
||||
...post(5, 'Start.Cut.2020.part1.rar', 3).slice(1),
|
||||
// Parts 1–2 seen, part 3 never: incomplete on the server.
|
||||
...post(500_000, 'Middle.Hole.2020.part1.rar', 3).slice(0, 2),
|
||||
], 600_000)
|
||||
// Chunks without its part 3: the hole file is given up (idle) long before the end.
|
||||
for (let at = 650_000; at < 950_000; at += 50_000) indexer.add([], at)
|
||||
// Parts 1–2 near the end, still open when the range ends: part 3 is in the next range.
|
||||
indexer.add(post(999_990, 'End.Cut.2020.part1.rar', 3).slice(0, 2), 1_000_001)
|
||||
indexer.finish()
|
||||
const batch = indexer.take()
|
||||
assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => [s[0], s[3]])]), [['Edge.File.2020.part1.rar', [[1, 5], [2, 6]]]])
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles]), [['incomplete', 2]])
|
||||
assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => s[0])]).sort(), [['End.Cut.2020.part1.rar', [1, 2]], ['Start.Cut.2020.part1.rar', [2, 3]]])
|
||||
assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]])
|
||||
assert.lengthOf(batch.releases, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user