voyager 3: the batch store writes releases once, by id references and bytea keys
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4e17e86f
commit
2580f20a43
32 files changed
+962
-497
No files matched your search
@@ -5,6 +5,8 @@ import { Spotter } from '#services/spotter'
|
||||
import { SpotInspector } from '#services/spot_inspector'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { parseSubject } from '#lib/headers/index'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import { insertRelease } from '#tests/functional/helpers/releases'
|
||||
|
||||
// Message ids whose sha1('<id>') starts with 0000: self-signed spots (key 7) pass on that hashcash alone.
|
||||
const SPOT = 'spottest731@spot.net'
|
||||
@@ -68,10 +70,7 @@ test.group('Spots', (group) => {
|
||||
// The release the header pass built for these files: the same identity, in the group.
|
||||
const parsed = parseSubject(subjects[0])!
|
||||
const identity = releaseIdentity({ poster, subject: parsed.base, name: parsed.name, weak: false })!
|
||||
const [r] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster, groups: '[]', group_id: groupId, files: 3, posted_at: posted, readable: true })
|
||||
.returning('id')
|
||||
const r = { id: await insertRelease({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster, groups: '[]', group_id: groupId, files: 3, posted_at: posted, readable: true }) }
|
||||
const releaseId = Number(r.id)
|
||||
const nzb = nzbXml(subjects, poster, posted.getTime() / 1000, 'nzbpw')
|
||||
await new Spotter().store(rawSpot(SPOT, 'Some.Movie.2024.1080p', 'Wachtwoord: geheim123'), SPOT, 10)
|
||||
@@ -100,13 +99,14 @@ test.group('Spots', (group) => {
|
||||
const inspector = new SpotInspector()
|
||||
assert.deepEqual(await inspector.inspectNzb(await db.from('spots').where('message_id', SPOT).firstOrFail(), nzb), { status: 'inspected', matched: 0 })
|
||||
assert.isTrue(await inspector.standalone(SPOT))
|
||||
const release = await db.from('releases').where('source', 'spot').firstOrFail()
|
||||
const release = await db.from('releases').where('source', SOURCES.spot).firstOrFail()
|
||||
assert.equal(release.name, 'Lonely File')
|
||||
// Its title says nothing: the spot's category (01, a09: a movie, x264).
|
||||
assert.equal(release.category, 2040)
|
||||
assert.equal(release.files, 1)
|
||||
assert.isNotNull(release.poster)
|
||||
assert.isNotNull(await db.from('posters').where('poster', release.poster).first())
|
||||
// Its poster (the spot's) by id, in posters for poster search.
|
||||
assert.isNotNull(release.poster_id)
|
||||
assert.isNotEmpty((await db.from('posters').where('id', release.poster_id).firstOrFail()).poster)
|
||||
const { default: NzbService } = await import('#services/nzb_service')
|
||||
assert.include((await NzbService.build(Number(release.id)))!.xml, 'lonely.file.mkv')
|
||||
// Not again while it has one; once the header pass builds the file's release, the spot moves there.
|
||||
@@ -114,10 +114,7 @@ test.group('Spots', (group) => {
|
||||
const group = await db.from('groups').where('name', 'alt.binaries.test.spots').firstOrFail()
|
||||
const parsed = parseSubject('[1/1] - "lonely.file.mkv" yEnc (1/9)')!
|
||||
const identity = releaseIdentity({ poster: 'p', subject: parsed.base, name: parsed.name, weak: false })!
|
||||
const [h] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster: 'p', groups: '[]', group_id: group.id, files: 1, posted_at: new Date(STAMP * 1000), readable: true })
|
||||
.returning('id')
|
||||
const h = { id: await insertRelease({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster: 'p', groups: '[]', group_id: group.id, files: 1, posted_at: new Date(STAMP * 1000), readable: true }) }
|
||||
assert.equal(await inspector.match(SPOT), 1)
|
||||
assert.isNull(await db.from('releases').where('id', release.id).first())
|
||||
assert.equal(Number((await db.from('spots').where('message_id', SPOT).firstOrFail()).release_id), Number(h.id))
|
||||
|
||||
Reference in new issue
Block a user