voyager 3: the batch store writes releases once, by id references and bytea keys
Test and deploy / test (push) Successful in 51s
Test and deploy / edge (push) Successful in 1m48s
Test and deploy / deploy (push) Skipped

Each release chunk is a release inserted once: release_key the SHA-1 of
the identity, the owner group and the first article (deterministic; no
numbering under a group lock), identity_key the identity's SHA-1 bytes,
poster_id and group_set_id through catalog_refs (cached per process),
source as a code (weak-stem added), the name into release_texts. A
finished release's candidates are claimable at once; a provisional tail
release is finished by a later chunk of its key, a finished one sent again
is left alone. Gone: joins into earlier releases and across groups, key
numbering, the group lock, the fragment merger. Interim: chunks, counts
and samples keyed by ranges.block_id, a range without a block gets one.

Readers follow: search (posters and group_sets joined, words through
release_texts, posters by id, source by code), the par2 merger, the
sidecar folder, the judge's scraped rule, the spot inspector, the spotter,
the NZB service, releases:regroup, edge:bench. Tests insert fixtures
through tests/functional/helpers/releases.ts; the suites pass again.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 21:04:03 +01:00
1 parent bc4e17e86f
commit 2580f20a43
32 files changed
+962 -497

No files matched your search

+6 -8
View File
@@ -2,6 +2,7 @@ import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import { Par2Pruner } from '#services/par2_pruner'
import { insertRelease } from '#tests/functional/helpers/releases'
test.group('Par2Pruner', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
@@ -12,10 +13,7 @@ test.group('Par2Pruner', (group) => {
const release = async () => {
n++
const key = `pp${String(n).padStart(6, '0')}`.padEnd(40, '0')
const [row] = await db
.table('releases')
.insert({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 2, readable: false })
.returning('id')
const row = { id: await insertRelease({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 2, readable: false }) }
return Number(row.id)
}
let candidate = 5000
@@ -59,12 +57,12 @@ test.group('Par2Pruner', (group) => {
const posted = new Date('2024-03-01T12:00:00Z')
const [group] = await db.table('groups').insert({ name: 'alt.binaries.pruned', priority: 1 }).returning('id')
const groupId = Number(group.id)
await db.from('releases').where('id', f).update({ group_id: groupId, posted_at: posted, last_posted_at: posted, first_article: 5000, last_article: 5100 })
// Finished (the window closed on it) long ago; its range done since.
await db.from('releases').where('id', f).update({ group_id: groupId, posted_at: posted, last_posted_at: posted, first_article: 5000, last_article: 5100, settled_at: old })
const [range] = await db
.table('ranges')
.insert({ group_id: groupId, start_id: 1, end_id: 10000, next_id: 10001, status: 'done', done_at: old, posted_low: new Date('2024-02-28T00:00:00Z'), posted_high: new Date('2024-03-03T00:00:00Z') })
.returning('id')
await db.table('range_blocks').insert({ group_id: groupId, start_id: 1, end_id: 10000, posted_low: new Date('2024-02-28T00:00:00Z'), posted_high: new Date('2024-03-03T00:00:00Z'), at_group_start: false })
// A lone candidate unprobed since before the range was done: its hash may still match. Kept.
await db.table('probe_candidates').insert({ id: 9001, release_id: b, kind: 'lone', message_id: '<9001@x>', file: 'x', status: 'waiting', created_at: new Date(old.getTime() - 3600_000) })
@@ -87,7 +85,7 @@ test.group('Par2Pruner', (group) => {
// A hash of its files since the merger's last window: the merger hasn't matched it yet. Kept.
const g = await release()
const unseen = await set('g', g, 1, null)
await db.from('releases').where('id', g).update({ group_id: groupId, posted_at: posted, last_posted_at: posted, first_article: 6000, last_article: 6100 })
await db.from('releases').where('id', g).update({ group_id: groupId, posted_at: posted, last_posted_at: posted, first_article: 6000, last_article: 6100, settled_at: old })
await db.table('worker_cursors').insert({ name: 'merger', at: new Date(Date.now() - 3600_000) })
await db.table('probe_candidates').insert({ id: 9002, release_id: await release(), kind: 'lone', message_id: '<9002@x>', file: 'x', status: 'done' })
await db.table('file_hashes').insert({ candidate_id: 9002, release_id: 1, hash16k: 'g0'.padEnd(32, '0'), size: 1000 })
@@ -105,7 +103,7 @@ test.group('Par2Pruner', (group) => {
let candidate = 7000
for (const letter of ['p', 'q', 'r']) {
const key = `pr${letter}`.padEnd(40, '0')
const [row] = await db.table('releases').insert({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, readable: false }).returning('id')
const row = { id: await insertRelease({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, readable: false }) }
const id = Number(row.id)
const setId = letter.repeat(32)
const h = `${letter}0`.padEnd(32, '0')