daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

voyager 3: the batch store writes releases once, by id references and bytea keys
Test and deploy / test (push) Successful in 51s
Details
Test and deploy / edge (push) Successful in 1m48s
Details
Test and deploy / deploy (push) Skipped
Details

Browse Source 
Each release chunk is a release inserted once: release_key the SHA-1 of
the identity, the owner group and the first article (deterministic; no
numbering under a group lock), identity_key the identity's SHA-1 bytes,
poster_id and group_set_id through catalog_refs (cached per process),
source as a code (weak-stem added), the name into release_texts. A
finished release's candidates are claimable at once; a provisional tail
release is finished by a later chunk of its key, a finished one sent again
is left alone. Gone: joins into earlier releases and across groups, key
numbering, the group lock, the fragment merger. Interim: chunks, counts
and samples keyed by ranges.block_id, a range without a block gets one.

Readers follow: search (posters and group_sets joined, words through
release_texts, posters by id, source by code), the par2 merger, the
sidecar folder, the judge's scraped rule, the spot inspector, the spotter,
the NZB service, releases:regroup, edge:bench. Tests insert fixtures
through tests/functional/helpers/releases.ts; the suites pass again.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 21:04:03 +01:00
1 parent bc4e17e86f
commit 2580f20a43
32 files changed
+962 -497
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/functional/par2_merger.spec.ts
+11 -34
View File
@@ -4,6 +4,7 @@ import testUtils from '@adonisjs/core/services/test_utils'
import { Par2Merger } from '#services/par2_merger'
import { encodeChunk } from '#lib/chunks/index'
import ChunkPartitions from '#services/chunk_partitions'
import { insertRelease } from '#tests/functional/helpers/releases'
const hash = (c: string) => c.repeat(32)
@@ -13,14 +14,11 @@ test.group('Par2Merger', (group) => {
test('merges the one-file releases of an obfuscated upload by its par2 set, named by the par2', async ({ assert }) => {
const at = new Date('2022-07-08T03:46:00Z')
const release = async (key: string, posted: Date, groupId: number, files = 1) => {
const [row] = await db
.table('releases')
.insert({
const row = { id: await insertRelease({
release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: groupId,
files, segments: files * 10, bytes: files * 1000, first_article: 100, last_article: 200, posted_at: posted, readable: false, file_total: 20,
settled_at: posted,
})
.returning('id')
}) }
return Number(row.id)
}
// The par2 index, two data files of the upload, the same file reposted a week later, and one in another group.
@@ -95,10 +93,7 @@ test.group('Par2Merger', (group) => {
const at = new Date('2022-07-08T03:46:00Z')
const ids: number[] = []
for (const key of ['dd000001', 'dd000002']) {
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false })
.returning('id')
const row = { id: await insertRelease({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false }) }
ids.push(Number(row.id))
}
// The par2 is held by the first; the second holds the file, and so do 20 releases elsewhere and years apart.
@@ -120,10 +115,7 @@ test.group('Par2Merger', (group) => {
const at = new Date('2022-07-08T03:46:00Z')
const ids: number[] = []
for (const [key, files] of [['bb000001', 1], ['bb000002', 1], ['bb000003', 3]] as const) {
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files, posted_at: at, readable: false })
.returning('id')
const row = { id: await insertRelease({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files, posted_at: at, readable: false }) }
ids.push(Number(row.id))
}
const setId = hash('7')
@@ -145,10 +137,7 @@ test.group('Par2Merger', (group) => {
const ids: number[] = []
for (let i = 1; i <= 6; i++) {
const key = `cc00000${i}`
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `kantodo${i}`, source: 'stem', groups: '[]', group_id: 1, files: 11, posted_at: at, readable: true })
.returning('id')
const row = { id: await insertRelease({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `kantodo${i}`, source: 'stem', groups: '[]', group_id: 1, files: 11, posted_at: at, readable: true }) }
ids.push(Number(row.id))
await db.table('par2_sets').insert({ set_id: setId, release_id: row.id, files: 1 })
}
@@ -163,10 +152,7 @@ test.group('Par2Merger', (group) => {
const release = async (poster: string, posted: Date, total: number, inner: string | null, groupId = 1) => {
n++
const key = `dd${String(n).padStart(6, '0')}`
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `hex${n}`, source: 'stem', groups: '[]', group_id: groupId, files: 1, posted_at: posted, readable: false, file_total: total, poster })
.returning('id')
const row = { id: await insertRelease({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: `hex${n}`, source: 'stem', groups: '[]', group_id: groupId, files: 1, posted_at: posted, readable: false, file_total: total, poster }) }
if (inner) {
await db.table('release_inspections').insert({ release_id: row.id, candidate_id: 5000 + n, kind: 'lone', result: JSON.stringify({ yencName: inner, type: '', size: 1 }) })
}
@@ -202,10 +188,7 @@ test.group('Par2Merger', (group) => {
const ids: number[] = []
for (let i = 1; i <= 3; i++) {
const key = `ee00000${i}`
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 10, posted_at: at, readable: false, file_total: 5, poster: 'p' })
.returning('id')
const row = { id: await insertRelease({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 10, posted_at: at, readable: false, file_total: 5, poster: 'p' }) }
ids.push(Number(row.id))
await db.table('release_inspections').insert({ release_id: row.id, candidate_id: 6000 + i, kind: 'lone', result: JSON.stringify({ yencName: `${stem}.7z.00${i}` }) })
}
@@ -217,10 +200,7 @@ test.group('Par2Merger', (group) => {
const at = new Date('2022-07-08T03:46:00Z')
const ids: number[] = []
for (const key of ['aa000001', 'aa000002', 'aa000003']) {
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false })
.returning('id')
const row = { id: await insertRelease({ release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, posted_at: at, readable: false }) }
ids.push(Number(row.id))
}
// Two test uploads whose 1 GiB files start alike, and a set describing one such file, held by the third.
@@ -242,13 +222,10 @@ test.group('Par2Merger', (group) => {
const release = async (name: string, files: number, total: number | null, posted: Date, chunkFiles: string[]) => {
n++
const key = `ff${String(n).padStart(6, '0')}`.padEnd(40, '0')
const [row] = await db
.table('releases')
.insert({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', group_id: 1, files, file_total: total, poster, posted_at: posted, readable: true, settled_at: posted })
.returning('id')
const row = { id: await insertRelease({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', group_id: 1, files, file_total: total, poster, posted_at: posted, readable: true, settled_at: posted }) }
const id = Number(row.id)
const payload = encodeChunk(chunkFiles.map((f, i) => ({ name: f, poster, subject: `${name} [${String(i).padStart(2, '0')}/${total ?? 0}] - "${f}" yEnc`, postedAt: posted.getTime(), groups: ['a.b.test'], segments: [[1, `${id}-${i}@x`, 1000]] })))
await db.table('release_chunks').insert({ release_id: id, range_id: 1, batch_from: 1, files: chunkFiles.length, segments: chunkFiles.length, payload })
await db.table('release_chunks').insert({ release_id: id, block_id: 1, batch_from: 1, files: chunkFiles.length, segments: chunkFiles.length, payload })
await db.table('fold_candidates').insert({ release_id: id }).onConflict('release_id').ignore()
return id
}