Daan MeijerandClaude Opus 5.5
b862201991
Files cut by a range's edge are put together once the ranges around them are done (coordinator:run, every minute): a complete file is judged by the rules and written as a release chunk with probe candidates; one still missing parts while both neighbouring ranges are done is dropped and counted (incomplete/edge). A neighbour not planned yet counts as not done, except before the group's oldest article. BatchStore.writeReleases is the shared write path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
66 lines
3.8 KiB
TypeScript
66 lines
3.8 KiB
TypeScript
import { test } from '@japa/runner'
|
||
import db from '@adonisjs/lucid/services/db'
|
||
import testUtils from '@adonisjs/core/services/test_utils'
|
||
import BatchStore from '#services/batch_store'
|
||
import FragmentMerger from '#services/fragment_merger'
|
||
import { RangeIndexer } from '#lib/edge/index'
|
||
import { decodeChunk } from '#lib/chunks/index'
|
||
import { DEFAULT_RULES } from '#lib/rules/index'
|
||
import type { Overview } from '#lib/nntp/index'
|
||
|
||
const GROUP = 'alt.binaries.test'
|
||
const rules = { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } }
|
||
|
||
function parts(name: string, total: number, numbers: number[], at: number): Overview[] {
|
||
return numbers.map((n, i) => [at + i, {
|
||
'subject': `Big.Upload.2020 [1/1] - "${name}" yEnc (${n}/${total})`, 'from': 'p <p@x>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
||
'message-id': `<${name}-${n}@test>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at + i}`,
|
||
}] as Overview)
|
||
}
|
||
|
||
/** Indexes one range's overviews and stores them as the range's only batch; the range ends done. */
|
||
async function indexRange(groupId: number, nodeId: number, start: number, end: number, overviews: Overview[]) {
|
||
const [r] = await db.table('ranges').insert({ group_id: groupId, start_id: start, end_id: end, next_id: start, status: 'leased', node_id: nodeId }).returning('id')
|
||
const indexer = new RangeIndexer({ group: GROUP, start, end, rules, edgeArticles: 100 })
|
||
indexer.add(overviews, end + 1)
|
||
indexer.finish()
|
||
await BatchStore.store(Number(r.id), nodeId, indexer.take())
|
||
await db.from('ranges').where('id', r.id).update({ status: 'done' })
|
||
return Number(r.id)
|
||
}
|
||
|
||
test.group('FragmentMerger', (group) => {
|
||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||
|
||
test('a file cut by a range edge is completed from both sides once both ranges are done', async ({ assert }) => {
|
||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1, low: 1 }).returning('id')
|
||
const [n] = await db.table('nodes').insert({ name: 'merge-node' }).returning('id')
|
||
// Parts 1–2 at the end of the first range (still open there), 3–4 at the start of the next.
|
||
await indexRange(g.id, n.id, 1, 1000, parts('big.part1.rar', 4, [1, 2], 990))
|
||
assert.equal((await db.from('fragments').count('* as c'))[0].c, '1')
|
||
assert.include(await FragmentMerger.run(), { completed: 0, incomplete: 0, waiting: 1 })
|
||
|
||
await indexRange(g.id, n.id, 1001, 2000, parts('big.part1.rar', 4, [3, 4], 1001))
|
||
assert.include(await FragmentMerger.run(), { completed: 1, incomplete: 0 })
|
||
assert.equal((await db.from('fragments').count('* as c'))[0].c, '0')
|
||
const release = await db.from('releases').first()
|
||
assert.include(release, { files: 1, chunks: 1 })
|
||
const chunk = await db.from('release_chunks').where('release_id', release.id).first()
|
||
assert.deepEqual(decodeChunk(chunk.payload)[0].segments.map((s) => s[0]), [1, 2, 3, 4])
|
||
assert.deepEqual((await db.from('probe_candidates')).map((c) => c.kind), ['archive'])
|
||
})
|
||
|
||
test('a file still missing parts when all its neighbours are done is dropped', async ({ assert }) => {
|
||
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1, low: 1 }).returning('id')
|
||
const [n] = await db.table('nodes').insert({ name: 'merge-node' }).returning('id')
|
||
await indexRange(g.id, n.id, 1, 1000, parts('gap.part1.rar', 5, [1, 2], 990))
|
||
await indexRange(g.id, n.id, 1001, 2000, parts('gap.part1.rar', 5, [4, 5], 1001))
|
||
// Its later neighbour isn't there yet: the missing part could still come.
|
||
assert.include(await FragmentMerger.run(), { completed: 0, incomplete: 0, waiting: 1 })
|
||
await indexRange(g.id, n.id, 2001, 3000, [])
|
||
assert.include(await FragmentMerger.run(), { completed: 0, incomplete: 1 })
|
||
assert.equal((await db.from('fragments').count('* as c'))[0].c, '0')
|
||
assert.lengthOf(await db.from('releases'), 0)
|
||
})
|
||
})
|