From bc4e17e86f9402c51eaedda24cdd17b9066c520b Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 25 Oct 2026 20:33:28 +0100 Subject: [PATCH] voyager 3: release_texts, the searchable strings of a release (migration 6200) One table and one trigram index for every string search matches against: the release name (kind 1, stored once more) and what anyone learns later (the prober's file names inside an archive, NFO titles, par2 set names, media titles), appended by insert, never an update of the releases row. Kinds and the per-kind cap in lib/release/text_kind.ts (textsToStore). The judge's dirty mark follows an insert or delete. Migration 6000 loses its two trigram indexes on releases. Co-Authored-By: Claude Fable 5.1 --- REBUILD.md | 29 ++++++++-- .../1770000006000_voyager3_catalog.ts | 10 +--- .../migrations/1770000006200_release_texts.ts | 35 +++++++++++ lib/release/text_kind.ts | 58 +++++++++++++++++++ tests/unit/release/text_kind.spec.ts | 42 ++++++++++++++ 5 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 database/migrations/1770000006200_release_texts.ts create mode 100644 lib/release/text_kind.ts create mode 100644 tests/unit/release/text_kind.spec.ts diff --git a/REBUILD.md b/REBUILD.md index cd8fd02..bb825d5 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -493,11 +493,27 @@ releases; the inspector matches them again), `nntp_shares`, `nodes`, `edge` (open at its block's end); `release_key` and `identity_key` as 20-byte `bytea`; `name`, `inner_name`. Gone: `poster`, `groups`, `last_seen_at`. Indexes: pkey, release_key unique, (identity_key, - group_id), name and inner-name trigram, newest, (poster_id, newest), - passworded, `(group_id, identity_key) WHERE edge` for the block merger, - `id WHERE settled_at IS NULL` for the provisional ones. The category - trigger reads the groups from `group_sets`; the judge's dirty trigger - watches `poster_id` instead of `poster`. + group_id), newest, (poster_id, newest), passworded, `(group_id, + identity_key) WHERE edge` for the block merger, `id WHERE settled_at IS + NULL` for the provisional ones. No trigram indexes: search matches + `release_texts`. The category trigger reads the groups from + `group_sets`; the judge's dirty trigger watches `poster_id` instead of + `poster`. +- `release_texts(release_id, kind smallint, text varchar(255))` (6200), + primary key on all three, one trigram index (fastupdate): every string + search matches against, appended by whoever learns one and never an + update of the releases row. Kinds (`lib/release/text_kind.ts`): 1 name + (the release name, stored once more so one index serves everything), + 2 file (the prober's names inside an archive), 3 nfo (an NFO's title), + 4 par2 (a set's name), 5 media (a container's title); at most 20 per + release and kind (`textsToStore`). A repeat insert is a no-op; the + judge's dirty mark follows an insert or delete, so a rule can react to a + name arriving later. Search: the trigram index yields release ids, joined + to `releases` by primary key (never an OR across the two tables, which + the planner cannot serve from indexes). `inner_name` stays on `releases` + as the one display string. Facets to filter on (resolution, codec, + language) are not texts: columns, or a `meta jsonb` with a + `jsonb_path_ops` index once one is proven useful. - `posters(id serial, poster unique)` with its trigram index; `group_sets(id serial, groups jsonb unique)`, a sorted array of names. - `blocks` (6100): `group_id`, `start_id`, `end_id`, `next_id` (the resume @@ -516,7 +532,8 @@ releases; the inspector matches them again), `nntp_shares`, `nodes`, the judge must read it by release or keep what it needs on the NVMe. Next: the batch store (write-once releases, posters and group sets by id, -bytea keys, blocks), then the window and albums in the Rust edge, block +bytea keys, blocks, the name into `release_texts`), then the window and +albums in the Rust edge, block leases and the block merger, the corpus replay harness. Until the batch store is done the 38 functional tests that write the old columns fail on `voyager-3`. diff --git a/database/migrations/1770000006000_voyager3_catalog.ts b/database/migrations/1770000006000_voyager3_catalog.ts index 98f65a7..d08a8bb 100644 --- a/database/migrations/1770000006000_voyager3_catalog.ts +++ b/database/migrations/1770000006000_voyager3_catalog.ts @@ -11,7 +11,8 @@ import env from '#start/env' * without its text copies (poster_id, group_set_id, the keys as 20-byte bytea, source a smallint), 8-byte * columns first, with `edge` (still open at its block's end, for the block merger) and settled_at as the finish * (null: a tail release still provisional). The category and judge triggers follow the new columns; - * releases_category_newest_idx (5.3 GB, used once) goes. + * releases_category_newest_idx (5.3 GB, used once) goes, and the trigram indexes: search matches release_texts + * (migration 6200), which holds the name and every searchable string learned later. * - fragments and range_blocks go with the fragment merger and the settler (the window finishes a release once); * release_final() goes: a release is final once finished. * - release_chunks, rule_counts and rule_samples are keyed by the block that wrote them (block_id; migration 6100). @@ -100,12 +101,7 @@ export default class extends BaseSchema { )`) this.schema.raw(`CREATE UNIQUE INDEX releases_release_key_unique ON releases (release_key)`) this.schema.raw(`CREATE INDEX releases_identity_idx ON releases (identity_key, group_id)`) - this.schema.raw( - `CREATE INDEX releases_name_trgm ON releases USING gin (name gin_trgm_ops) WITH (fastupdate = on, gin_pending_list_limit = 65536)` - ) - this.schema.raw( - `CREATE INDEX releases_inner_name_trgm ON releases USING gin (inner_name gin_trgm_ops) WHERE inner_name IS NOT NULL` - ) + // No trigram indexes here: search matches release_texts (migration 6200), where the name is stored once more. this.schema.raw( `CREATE INDEX releases_newest_idx ON releases (posted_at DESC NULLS LAST, id DESC)` ) diff --git a/database/migrations/1770000006200_release_texts.ts b/database/migrations/1770000006200_release_texts.ts new file mode 100644 index 0000000..24253e1 --- /dev/null +++ b/database/migrations/1770000006200_release_texts.ts @@ -0,0 +1,35 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * release_texts (REBUILD.md, "Voyager 3 … Schema"): every string search matches against, one table, one trigram + * index. The release name goes in (kind 1) with whatever anyone learns later: the prober's file names inside an + * archive, NFO titles, par2 set names, media titles (lib/release/text_kind.ts). Appending is an insert, never an + * update of the wide releases row; a repeat insert is a no-op by the primary key; the judge's dirty mark follows + * (judge_dirty_by_release_id, migration 4900), so a rule can react to a name arriving later. On the NVMe: search + * reads it. releases keeps name and inner_name for display, without trigram indexes (migration 6000). + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(` + CREATE TABLE release_texts ( + release_id bigint NOT NULL, + -- lib/release/text_kind.ts: 1 name, 2 file, 3 nfo, 4 par2, 5 media. + kind smallint NOT NULL, + text varchar(255) NOT NULL, + PRIMARY KEY (release_id, kind, text) + )`) + this.schema.raw( + `CREATE INDEX release_texts_trgm ON release_texts USING gin (text gin_trgm_ops) WITH (fastupdate = on, gin_pending_list_limit = 65536)` + ) + this.schema.raw( + `CREATE TRIGGER judge_dirty_insert AFTER INSERT ON release_texts REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()` + ) + this.schema.raw( + `CREATE TRIGGER judge_dirty_delete AFTER DELETE ON release_texts REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION judge_dirty_by_release_id()` + ) + } + + async down() { + this.schema.raw(`DROP TABLE IF EXISTS release_texts`) + } +} diff --git a/lib/release/text_kind.ts b/lib/release/text_kind.ts new file mode 100644 index 0000000..d3e8f2a --- /dev/null +++ b/lib/release/text_kind.ts @@ -0,0 +1,58 @@ +/** + * What a searchable string of a release is, as release_texts.kind stores it (a smallint; voyager 3, migration 6200). + * The table holds every string search matches against (the release name too), appended by whoever learns one: the + * batch store the name, the prober the file names inside an archive, later passes NFO titles, par2 set names, + * media titles. The numbers never change meaning once written. + */ +export const TEXT_KINDS = { + /** The release name (releases.name, once more for the one trigram index). */ + name: 1, + /** A file name inside an archive (the prober's inner names). */ + file: 2, + /** A title read from an NFO. */ + nfo: 3, + /** A par2 set's name when it differs from the release name. */ + par2: 4, + /** A title from a media container's metadata. */ + media: 5, +} as const + +export type TextKind = keyof typeof TEXT_KINDS + +/** The longest string stored (release_texts.text is a varchar(255)); longer ones are cut, not refused. */ +export const TEXT_MAX = 255 +/** The most strings one writer appends per release and kind (an archive of two thousand files is not two thousand rows). */ +export const TEXTS_PER_KIND = 20 + +const NAMES = Object.fromEntries( + Object.entries(TEXT_KINDS).map(([name, code]) => [code, name]) +) as Record + +/** The code of a kind; throws for an unknown one (a typo would write a wrong number for good). */ +export function textKindCode(name: string): number { + const code = TEXT_KINDS[name as TextKind] + if (code === undefined) throw new Error(`Unknown release text kind "${name}"`) + return code +} + +/** The name of a stored code; "unknown" for one not listed. */ +export function textKindName(code: number | string | null | undefined): TextKind | 'unknown' { + return NAMES[Number(code)] ?? 'unknown' +} + +/** + * The strings a writer appends for one kind: trimmed, cut to TEXT_MAX, empty ones and repeats dropped, the first + * TEXTS_PER_KIND kept in the order given (the writer puts the most telling first). + */ +export function textsToStore(texts: Iterable): string[] { + const seen = new Set() + const out: string[] = [] + for (const raw of texts) { + const text = (raw ?? '').trim().slice(0, TEXT_MAX) + if (!text || seen.has(text)) continue + seen.add(text) + out.push(text) + if (out.length === TEXTS_PER_KIND) break + } + return out +} diff --git a/tests/unit/release/text_kind.spec.ts b/tests/unit/release/text_kind.spec.ts new file mode 100644 index 0000000..16a3273 --- /dev/null +++ b/tests/unit/release/text_kind.spec.ts @@ -0,0 +1,42 @@ +import { test } from '@japa/runner' +import { + TEXT_KINDS, + TEXTS_PER_KIND, + TEXT_MAX, + textKindCode, + textKindName, + textsToStore, +} from '#lib/release/text_kind' + +test.group('release text kinds', () => { + test('names and codes map both ways; the codes are fixed', ({ assert }) => { + assert.deepEqual(TEXT_KINDS, { name: 1, file: 2, nfo: 3, par2: 4, media: 5 }) + for (const [name, code] of Object.entries(TEXT_KINDS)) { + assert.equal(textKindCode(name), code) + assert.equal(textKindName(code), name) + assert.equal(textKindName(String(code)), name) + } + }) + + test('an unknown name throws, an unknown code reads as unknown', ({ assert }) => { + assert.throws(() => textKindCode('title'), /Unknown release text kind/) + assert.equal(textKindName(99), 'unknown') + assert.equal(textKindName(null), 'unknown') + }) + + test('textsToStore trims, cuts, drops empties and repeats, keeps the first TEXTS_PER_KIND', ({ + assert, + }) => { + const long = 'x'.repeat(TEXT_MAX + 10) + assert.deepEqual(textsToStore([' a.mkv ', '', null, 'a.mkv', 'b.mkv', long]), [ + 'a.mkv', + 'b.mkv', + 'x'.repeat(TEXT_MAX), + ]) + const many = Array.from({ length: TEXTS_PER_KIND + 5 }, (_, i) => `file${i}.rar`) + const kept = textsToStore(many) + assert.lengthOf(kept, TEXTS_PER_KIND) + assert.equal(kept[0], 'file0.rar') + assert.equal(kept[TEXTS_PER_KIND - 1], `file${TEXTS_PER_KIND - 1}.rar`) + }) +})