voyager 3: the batch store writes releases once, by id references and bytea keys
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4e17e86f
commit
2580f20a43
32 files changed
+962
-497
No files matched your search
@@ -9,6 +9,9 @@ import { decodeSpotBinary } from '#lib/spotnet/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/headers/index'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import CatalogRefs from '#services/catalog_refs'
|
||||
|
||||
/** Spots claimed per round, newest first. */
|
||||
const BATCH = 20
|
||||
@@ -251,8 +254,8 @@ export class SpotInspector {
|
||||
}
|
||||
if (wanted.length === 0) return 0
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
`SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
|
||||
for (const r of rows) {
|
||||
@@ -275,7 +278,7 @@ export class SpotInspector {
|
||||
await db.transaction(async (trx) => {
|
||||
// A release of the spot's own goes now that the header pass built its files' releases.
|
||||
if (spot.release_id) {
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', 'spot').select('id').first()
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
|
||||
if (own) {
|
||||
await trx.from('release_spots').where('release_id', own.id).delete()
|
||||
await trx.from('release_passwords').where('release_id', own.id).delete()
|
||||
@@ -335,16 +338,19 @@ export class SpotInspector {
|
||||
const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
|
||||
const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
|
||||
const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
|
||||
// Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
|
||||
const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
|
||||
const groupSetId = await CatalogRefs.groupSetId(db, groups)
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
release_key: `spot:${key}`,
|
||||
identity_key: key,
|
||||
release_key: releaseKey(key, groupId, null),
|
||||
identity_key: identityKey(key),
|
||||
group_id: groupId,
|
||||
group_set_id: groupSetId,
|
||||
poster_id: posterId,
|
||||
name: spot.title.slice(0, 1000),
|
||||
source: 'spot',
|
||||
poster: spot.poster,
|
||||
groups: JSON.stringify(groups),
|
||||
source: SOURCES.spot,
|
||||
posted_at: spot.posted_at,
|
||||
last_posted_at: spot.posted_at,
|
||||
files: files.length,
|
||||
@@ -363,8 +369,8 @@ export class SpotInspector {
|
||||
const releaseId = Number(row.id)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
|
||||
// Its poster for poster search, as the batch store adds its releases' (migration 4000).
|
||||
if (spot.poster) await trx.rawQuery(`INSERT INTO posters (poster) VALUES (left(?, 200)) ON CONFLICT DO NOTHING`, [spot.poster])
|
||||
// Its title, searchable (release_texts).
|
||||
await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
|
||||
if (spot.password) {
|
||||
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user