voyager 3: the batch store writes releases once, by id references and bytea keys
Test and deploy / test (push) Successful in 51s
Test and deploy / edge (push) Successful in 1m48s
Test and deploy / deploy (push) Skipped

Each release chunk is a release inserted once: release_key the SHA-1 of
the identity, the owner group and the first article (deterministic; no
numbering under a group lock), identity_key the identity's SHA-1 bytes,
poster_id and group_set_id through catalog_refs (cached per process),
source as a code (weak-stem added), the name into release_texts. A
finished release's candidates are claimable at once; a provisional tail
release is finished by a later chunk of its key, a finished one sent again
is left alone. Gone: joins into earlier releases and across groups, key
numbering, the group lock, the fragment merger. Interim: chunks, counts
and samples keyed by ranges.block_id, a range without a block gets one.

Readers follow: search (posters and group_sets joined, words through
release_texts, posters by id, source by code), the par2 merger, the
sidecar folder, the judge's scraped rule, the spot inspector, the spotter,
the NZB service, releases:regroup, edge:bench. Tests insert fixtures
through tests/functional/helpers/releases.ts; the suites pass again.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 21:04:03 +01:00
1 parent bc4e17e86f
commit 2580f20a43
32 files changed
+962 -497

No files matched your search

+16 -10
View File
@@ -9,6 +9,9 @@ import { decodeSpotBinary } from '#lib/spotnet/index'
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
import { parseSubject } from '#lib/headers/index'
import { releaseIdentity } from '#lib/release/index'
import { identityKey, releaseKey } from '#lib/release/keys'
import { SOURCES } from '#lib/release/source'
import CatalogRefs from '#services/catalog_refs'
/** Spots claimed per round, newest first. */
const BATCH = 20
@@ -251,8 +254,8 @@ export class SpotInspector {
}
if (wanted.length === 0) return 0
const { rows } = await db.rawQuery(
`SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
[[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
`SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
[[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
)
const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
for (const r of rows) {
@@ -275,7 +278,7 @@ export class SpotInspector {
await db.transaction(async (trx) => {
// A release of the spot's own goes now that the header pass built its files' releases.
if (spot.release_id) {
const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
if (own) {
await trx.from('release_spots').where('release_id', own.id).delete()
await trx.from('release_passwords').where('release_id', own.id).delete()
@@ -335,16 +338,19 @@ export class SpotInspector {
const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
// Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
const groupSetId = await CatalogRefs.groupSetId(db, groups)
const [row] = await db
.table('releases')
.insert({
release_key: `spot:${key}`,
identity_key: key,
release_key: releaseKey(key, groupId, null),
identity_key: identityKey(key),
group_id: groupId,
group_set_id: groupSetId,
poster_id: posterId,
name: spot.title.slice(0, 1000),
source: 'spot',
poster: spot.poster,
groups: JSON.stringify(groups),
source: SOURCES.spot,
posted_at: spot.posted_at,
last_posted_at: spot.posted_at,
files: files.length,
@@ -363,8 +369,8 @@ export class SpotInspector {
const releaseId = Number(row.id)
await db.transaction(async (trx) => {
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
// Its poster for poster search, as the batch store adds its releases' (migration 4000).
if (spot.poster) await trx.rawQuery(`INSERT INTO posters (poster) VALUES (left(?, 200)) ON CONFLICT DO NOTHING`, [spot.poster])
// Its title, searchable (release_texts).
await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
if (spot.password) {
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
}