Daan MeijerandClaude Fable 5.1
2580f20a43
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
68 lines
4.4 KiB
TypeScript
68 lines
4.4 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import BatchStore from '#services/batch_store'
|
|
import { decidePar2Completeness, par2Complete } from '#services/par2_completeness'
|
|
import { RangeIndexer } from '#lib/edge/index'
|
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
|
import type { Overview } from '#lib/nntp/index'
|
|
|
|
const GROUP = 'alt.binaries.test'
|
|
|
|
/** A file posted without an [i/N] counter: the release gets no file_total. */
|
|
function file(name: string, parts: number, at: number): Overview[] {
|
|
return Array.from({ length: parts }, (_, i) => [at + i, {
|
|
'subject': `"${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
|
'message-id': `<${name}-${i + 1}@test>`, ':bytes': '700000', 'xref': `news.example ${GROUP}:${at + i}`,
|
|
}] as Overview)
|
|
}
|
|
|
|
async function store(start: number, overviews: Overview[]) {
|
|
const g = (await db.from('groups').where('name', GROUP).first()) ?? (await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('*'))[0]
|
|
const [n] = await db.table('nodes').insert({ name: `n${start}` }).returning('id')
|
|
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: start + 999, next_id: start, status: 'leased', node_id: n.id }).returning('id')
|
|
const indexer = new RangeIndexer({ group: GROUP, start, end: start + 999, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
|
indexer.add(overviews, start + 1000)
|
|
indexer.finish()
|
|
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
|
}
|
|
|
|
const setFile = (name: string, length: number, hash16k = name.padEnd(32, '0').slice(0, 32)) => ({ name, length, hash16k })
|
|
|
|
test.group('par2 completeness', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('every described file by name or by hash; a renamed upload by its files and bytes', ({ assert }) => {
|
|
const set = [setFile('a.mkv', 1000), setFile('a.nfo', 10)]
|
|
const files = [{ name: 'a.mkv', size: 1100 }, { name: 'a.nfo', size: 12 }, { name: 'a.par2', size: 50 }]
|
|
assert.isTrue(par2Complete(set, files, []))
|
|
assert.isFalse(par2Complete(set, files.slice(0, 1), []))
|
|
// The nfo renamed, but its probe hashed it.
|
|
assert.isTrue(par2Complete(set, [files[0], { name: 'x.nfo', size: 12 }], [{ hash16k: set[1].hash16k, size: 10 }]))
|
|
// Renamed after the par2 was made (random names): two data files and enough bytes; one missing, or too small.
|
|
assert.isTrue(par2Complete(set, [{ name: 'Qx81.bin', size: 1100 }, { name: 'Zr02.bin', size: 12 }, { name: 'Qx.par2', size: 50 }], []))
|
|
assert.isFalse(par2Complete(set, [{ name: 'Qx81.bin', size: 1100 }, { name: 'Qx.par2', size: 50 }], []))
|
|
assert.isFalse(par2Complete(set, [{ name: 'Qx81.bin', size: 500 }, { name: 'Zr02.bin', size: 12 }], []))
|
|
})
|
|
|
|
test('decided once finished, by the par2 set the release holds', async ({ assert }) => {
|
|
await store(1, [...file('Some.Upload.mkv', 3, 10), ...file('Some.Upload.par2', 1, 20)])
|
|
const release = await db.from('releases').firstOrFail()
|
|
assert.isNull(release.file_total)
|
|
await db.table('par2_files').multiInsert([setFile('Some.Upload.mkv', 2_000_000), setFile('Some.Upload.nfo', 100)].map((f, i) => ({ set_id: 's'.repeat(32), file_id: String(i).repeat(32), ...f })))
|
|
await db.table('par2_sets').insert({ set_id: 's'.repeat(32), release_id: release.id, files: 2 })
|
|
// A provisional tail release (not finished yet): left alone.
|
|
await db.from('releases').where('id', release.id).update({ settled_at: null })
|
|
assert.equal(await decidePar2Completeness(db, [release.id]), 0)
|
|
await db.from('releases').where('id', release.id).update({ settled_at: new Date() })
|
|
assert.equal(await decidePar2Completeness(db, [release.id]), 1)
|
|
assert.isFalse((await db.from('releases').where('id', release.id).firstOrFail()).complete)
|
|
// Another upload with the nfo: complete.
|
|
await store(1001, [...file('Some.Upload.mkv', 3, 1010), ...file('Some.Upload.par2', 1, 1020), ...file('Some.Upload.nfo', 1, 1030)])
|
|
const whole = await db.from('releases').whereNot('id', release.id).firstOrFail()
|
|
await db.table('par2_sets').insert({ set_id: 's'.repeat(32), release_id: whole.id, files: 2 })
|
|
assert.equal(await decidePar2Completeness(db, [whole.id]), 1)
|
|
assert.isTrue((await db.from('releases').where('id', whole.id).firstOrFail()).complete)
|
|
})
|
|
})
|