Daan MeijerandClaude Fable 5.1
2580f20a43
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
94 lines
3.3 KiB
TypeScript
94 lines
3.3 KiB
TypeScript
import type db from '@adonisjs/lucid/services/db'
|
|
|
|
type Client = Pick<typeof db, 'rawQuery'>
|
|
|
|
/** Posters are kept to this length (as the trigram index and poster search always did; a btree entry holds 2,704 bytes). */
|
|
export const POSTER_MAX = 200
|
|
/** Cached ids per process: group sets are ~2,600 in all; posters many more, so the cache is emptied when it grows past this. */
|
|
const CACHE_MAX = 200_000
|
|
|
|
const posterCache = new Map<string, number>()
|
|
const groupSetCache = new Map<string, number>()
|
|
|
|
/** A group list as group_sets stores it: sorted, without repeats, as JSON. */
|
|
export function groupSetKey(groups: string[]): string {
|
|
return JSON.stringify([...new Set(groups)].sort())
|
|
}
|
|
|
|
/** A poster as posters stores it. */
|
|
export function posterText(poster: string): string {
|
|
return poster.slice(0, POSTER_MAX)
|
|
}
|
|
|
|
async function ids(
|
|
client: Client,
|
|
cache: Map<string, number>,
|
|
table: 'posters' | 'group_sets',
|
|
column: 'poster' | 'groups',
|
|
cast: 'text' | 'jsonb',
|
|
values: string[]
|
|
): Promise<Map<string, number>> {
|
|
const out = new Map<string, number>()
|
|
const missing: string[] = []
|
|
for (const v of values) {
|
|
const id = cache.get(v)
|
|
if (id === undefined) missing.push(v)
|
|
else out.set(v, id)
|
|
}
|
|
if (missing.length === 0) return out
|
|
// Insert what's new, then read every id (ON CONFLICT DO NOTHING returns nothing for the rows that existed).
|
|
await client.rawQuery(
|
|
`INSERT INTO ${table} (${column}) SELECT DISTINCT v::${cast} FROM unnest(?::text[]) AS v ON CONFLICT DO NOTHING`,
|
|
[missing]
|
|
)
|
|
const { rows } = await client.rawQuery(
|
|
`SELECT id, ${column}::text AS value FROM ${table} WHERE ${column} = ANY(?::${cast}[])`,
|
|
[missing]
|
|
)
|
|
if (cache.size + rows.length > CACHE_MAX) cache.clear()
|
|
for (const r of rows) {
|
|
// jsonb comes back in Postgres' spelling ("["a", "b"]"); the caller's key is the JSON we sent.
|
|
const key = cast === 'jsonb' ? JSON.stringify(JSON.parse(r.value)) : r.value
|
|
cache.set(key, Number(r.id))
|
|
out.set(key, Number(r.id))
|
|
}
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* Ids in the catalog's lookup tables (voyager 3, migration 6000): a release's poster_id and group_set_id. New
|
|
* values are inserted; known ones come from a cache per process (the batch store sees the same posters and
|
|
* group sets batch after batch).
|
|
*/
|
|
export default {
|
|
/** Ids by poster (posterText applied), for a batch of posters. */
|
|
async posterIds(client: Client, posters: string[]): Promise<Map<string, number>> {
|
|
return ids(client, posterCache, 'posters', 'poster', 'text', [
|
|
...new Set(posters.map(posterText)),
|
|
])
|
|
},
|
|
|
|
async posterId(client: Client, poster: string): Promise<number> {
|
|
const found = await this.posterIds(client, [poster])
|
|
return found.get(posterText(poster))!
|
|
},
|
|
|
|
/** Ids by groupSetKey, for a batch of group lists. */
|
|
async groupSetIds(client: Client, sets: string[][]): Promise<Map<string, number>> {
|
|
return ids(client, groupSetCache, 'group_sets', 'groups', 'jsonb', [
|
|
...new Set(sets.map(groupSetKey)),
|
|
])
|
|
},
|
|
|
|
async groupSetId(client: Client, groups: string[]): Promise<number> {
|
|
const found = await this.groupSetIds(client, [groups])
|
|
return found.get(groupSetKey(groups))!
|
|
},
|
|
|
|
/** Forgets the cached ids (tests roll their transactions back, so the ids they saw are gone). */
|
|
forget() {
|
|
posterCache.clear()
|
|
groupSetCache.clear()
|
|
},
|
|
}
|