Daan MeijerandClaude Opus 5.5
20910647fa
A batch is stored in one transaction and only at its range's resume point, so a retried batch can't count twice. Garbage is judged before per-article: a dropped tool isn't worth reading bodies for either. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
4.1 KiB
TypeScript
77 lines
4.1 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import BatchStore, { StaleBatch } from '#services/batch_store'
|
|
import { RangeIndexer } from '#lib/edge/index'
|
|
import { decodeChunk } from '#lib/chunks/index'
|
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
|
import type { Overview } from '#lib/nntp/index'
|
|
|
|
const GROUP = 'alt.binaries.test'
|
|
let id = 0
|
|
|
|
function post(at: number, name: string, parts: number, counter: string, tool: 'ngPost' | 'plain' = 'plain'): Overview[] {
|
|
return Array.from({ length: parts }, (_, i) => {
|
|
id++
|
|
const mid = tool === 'ngPost' ? `${id.toString(16).padStart(32, '0')}@ngPost` : `art${id}@test`
|
|
return [at + i, { 'subject': `${counter} - "${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<${mid}>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at + i}` }] as Overview
|
|
})
|
|
}
|
|
|
|
async function leasedRange() {
|
|
const [group] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
|
const [node] = await db.table('nodes').insert({ name: 'test-node' }).returning('id')
|
|
const [range] = await db
|
|
.table('ranges')
|
|
.insert({ group_id: group.id, start_id: 1, end_id: 1_000_000, next_id: 1, status: 'leased', node_id: node.id, rule_version: 1 })
|
|
.returning('id')
|
|
return { rangeId: Number(range.id), nodeId: Number(node.id) }
|
|
}
|
|
|
|
function indexer() {
|
|
return new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } }, edgeArticles: 1000 })
|
|
}
|
|
|
|
test.group('BatchStore', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('stores releases, their chunks, probe candidates, rule counts and fragments, and moves the resume point', async ({ assert }) => {
|
|
const { rangeId, nodeId } = await leasedRange()
|
|
const ix = indexer()
|
|
ix.add([
|
|
...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 2, 'Some.Show.S01E01 [1/2]'),
|
|
...post(100_010, 'Some.Show.S01E01.1080p.WEB.par2', 1, 'Some.Show.S01E01 [2/2]'),
|
|
...post(100_020, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 3, '[1/1]', 'ngPost'),
|
|
...post(10, 'Edge.Cut.2020.part1.rar', 3, 'Edge.Cut [1/1]').slice(0, 1),
|
|
], 500_000)
|
|
const first = ix.take()
|
|
const stored = await BatchStore.store(rangeId, nodeId, first)
|
|
assert.include(stored, { releases: 1, newReleases: 1, candidates: 2, fragments: 0 })
|
|
|
|
const release = await db.from('releases').first()
|
|
assert.include(release, { files: 2, segments: '3', chunks: 1, readable: true, has_par2: true, file_total: 2 })
|
|
const chunk = await db.from('release_chunks').where('release_id', release.id).first()
|
|
assert.deepEqual(decodeChunk(chunk.payload).map((f) => f.name).sort(), ['Some.Show.S01E01.1080p.WEB.par2', 'Some.Show.S01E01.1080p.WEB.part1.rar'])
|
|
assert.deepEqual((await db.from('probe_candidates').orderBy('kind')).map((c) => c.kind), ['archive', 'par2'])
|
|
assert.deepEqual((await db.from('rule_counts')).map((c) => [c.rule, c.key, Number(c.articles)]), [['garbage', 'ngPost alt.binaries.test', 3]])
|
|
assert.equal(Number((await db.from('ranges').where('id', rangeId).first()).next_id), 500_000)
|
|
|
|
// The same batch again (a retry after a lost reply) is refused, and changes nothing.
|
|
await assert.rejects(() => BatchStore.store(rangeId, nodeId, first), StaleBatch)
|
|
assert.equal((await db.from('releases').first()).files, 2)
|
|
|
|
// The end of the range: the cut-off file near the start edge is kept as a fragment.
|
|
ix.finish()
|
|
const last = ix.take()
|
|
assert.include(await BatchStore.store(rangeId, nodeId, last), { fragments: 1 })
|
|
assert.equal(Number((await db.from('ranges').where('id', rangeId).first()).next_id), 1_000_001)
|
|
})
|
|
|
|
test('a range leased to another node is refused', async ({ assert }) => {
|
|
const { rangeId, nodeId } = await leasedRange()
|
|
const ix = indexer()
|
|
ix.add(post(100_000, 'A.Release.2020.mkv', 1, 'A [1/1]'), 200_000)
|
|
await assert.rejects(() => BatchStore.store(rangeId, nodeId + 1, ix.take()), /not leased to this node/)
|
|
})
|
|
})
|