voyager 3: the batch store writes releases once, by id references and bytea keys
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4e17e86f
commit
2580f20a43
32 files changed
+962
-497
No files matched your search
@@ -0,0 +1,57 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import CatalogRefs from '#services/catalog_refs'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
|
||||
/** A release as the tests describe one: text keys, a poster and groups by name, the source by name. */
|
||||
export interface ReleaseFixture {
|
||||
/** Any text; the stored key is its hash (keyOf), or 40 hex characters decoded. Default: from the identity, group and first article. */
|
||||
release_key?: string
|
||||
/** Any text (default: release_key, else the name). */
|
||||
identity_key?: string
|
||||
name: string
|
||||
source?: string
|
||||
poster?: string | null
|
||||
groups?: string[] | string
|
||||
group_id?: number | null
|
||||
first_article?: number | null
|
||||
[column: string]: unknown
|
||||
}
|
||||
|
||||
/** The bytes a text key is stored as (identityKey: 40 hex characters decoded, anything else hashed). */
|
||||
export const keyOf = identityKey
|
||||
|
||||
/** Inserts releases with their poster, group set, keys and source as the catalog stores them; their ids in order. */
|
||||
export async function insertReleases(fixtures: ReleaseFixture[]): Promise<number[]> {
|
||||
const ids: number[] = []
|
||||
for (const f of fixtures) {
|
||||
const { release_key: textKey, identity_key: textIdentity, source, poster, groups, ...rest } = f
|
||||
const identity = textIdentity ?? textKey ?? f.name
|
||||
const groupList: string[] = typeof groups === 'string' ? JSON.parse(groups) : (groups ?? [])
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
...rest,
|
||||
release_key: textKey
|
||||
? identityKey(textKey)
|
||||
: releaseKey(identity, f.group_id ?? null, f.first_article ?? null),
|
||||
identity_key: identityKey(identity),
|
||||
source: sourceCode(source ?? 'stem'),
|
||||
poster_id: poster ? await CatalogRefs.posterId(db, poster) : null,
|
||||
group_set_id: await CatalogRefs.groupSetId(db, groupList),
|
||||
})
|
||||
.returning('id')
|
||||
const id = Number(row.id)
|
||||
await db.rawQuery(
|
||||
`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`,
|
||||
[id, f.name]
|
||||
)
|
||||
ids.push(id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
export async function insertRelease(fixture: ReleaseFixture): Promise<number> {
|
||||
const [id] = await insertReleases([fixture])
|
||||
return id
|
||||
}
|
||||
Reference in new issue
Block a user