usenet-indexer/database/migrations/1768620765060_search_text_and_index_cleanup.ts

import { BaseSchema } from '@adonisjs/lucid/schema'

const VISIBLE = `(file_count > 0 OR source = 'spot')`
const BATCH = 20_000

/**
 * Search and index cleanup (CONCEPT.md D33).
 * - `releases.search_text`: name, archive listing and readable file-name
 *   stems, normalised (lowercase; dots, dashes, underscores and spaces as
 *   one space), kept by triggers; one trigram index replaces the four the
 *   old per-table OR search could hardly use.
 * - Spot status indexes become partial indexes over the few rows the worker
 *   and search look for.
 * - Unused release indexes go (size: never scanned; complete: a boolean).
 * Runs online: indexes are built and dropped concurrently.
 */
export default class extends BaseSchema {
  static disableTransactions = true

  async up() {
    const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any)
    // For IndexMaintenance; needs a superuser (it is one in the compose setup).
    await q(`CREATE EXTENSION IF NOT EXISTS pgstattuple`).catch((error) => console.warn(`pgstattuple not installed: ${error.message}`))

    // Spots: the worker only looks for pending/processing and un-detailed imported spots, search for hidden ones.
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_todo_idx ON spots (article_number) WHERE status IN ('pending', 'processing')`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_hidden_release_idx ON spots (release_id) WHERE status IN ('removed', 'dummy')`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_details_todo_idx ON spots (posted_at DESC) WHERE status = 'imported' AND details_at IS NULL`)
    for (const index of ['spots_status_posted_idx', 'spots_status_article_idx', 'spots_status_created_at_index', 'spots_status_release_id_index']) {
      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
    }
    for (const index of ['releases_size_index', 'releases_complete_idx']) {
      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
    }

    // Search text.
    await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS search_files text, ADD COLUMN IF NOT EXISTS search_text text`)
    await q(`
      CREATE OR REPLACE FUNCTION search_normalize(t text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
        SELECT btrim(lower(regexp_replace(coalesce(t, ''), '[._[:space:]-]+', ' ', 'g')))
      $$`)
    await q(`
      CREATE OR REPLACE FUNCTION release_search_text(name text, contents jsonb, files text) RETURNS text IMMUTABLE LANGUAGE sql AS $$
        SELECT left(concat_ws(' ',
          search_normalize(name),
          nullif(search_normalize((SELECT string_agg(e->>'name', ' ') FROM (
            SELECT e FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) e LIMIT 200) x)), ''),
          nullif(files, '')), 8000)
      $$`)
    // Visible releases only; computed when one becomes visible or its name, listing or file names change.
    await q(`
      CREATE OR REPLACE FUNCTION releases_search_text() RETURNS trigger LANGUAGE plpgsql AS $$
      BEGIN
        IF NOT (NEW.file_count > 0 OR NEW.source = 'spot') THEN
          NEW.search_text := NULL;
        ELSIF TG_OP = 'INSERT' OR NEW.search_text IS NULL
          OR NEW.name IS DISTINCT FROM OLD.name OR NEW.contents IS DISTINCT FROM OLD.contents
          OR NEW.search_files IS DISTINCT FROM OLD.search_files THEN
          NEW.search_text := release_search_text(NEW.name, NEW.contents, NEW.search_files);
        END IF;
        RETURN NEW;
      END $$`)
    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
    await q(`CREATE TRIGGER releases_search_text BEFORE INSERT OR UPDATE ON releases FOR EACH ROW EXECUTE FUNCTION releases_search_text()`)
    // A file's readable stem joins its release's search text when it adds something.
    await q(`
      CREATE OR REPLACE FUNCTION files_release_search() RETURNS trigger LANGUAGE plpgsql AS $$
      DECLARE
        n text := coalesce(NEW.real_name, NEW.filename);
        s text;
      BEGIN
        IF NEW.release_id IS NULL OR NOT is_readable_name(n) THEN RETURN NULL; END IF;
        s := search_normalize(name_stem(n));
        IF s = '' THEN RETURN NULL; END IF;
        UPDATE releases SET search_files = left(concat_ws(' ', search_files, s), 4000)
        WHERE id = NEW.release_id AND position(s IN coalesce(search_text, search_normalize(name))) = 0
          AND length(coalesce(search_files, '')) < 4000;
        RETURN NULL;
      END $$`)
    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
    await q(`CREATE TRIGGER files_release_search AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_search()`)

    // Fill: readable file stems per release (one pass over files), then every visible release in batches.
    await q(`DROP TABLE IF EXISTS search_stems_tmp`)
    await q(`
      CREATE UNLOGGED TABLE search_stems_tmp AS
      SELECT release_id, left(string_agg(DISTINCT s, ' '), 4000) AS stems FROM (
        SELECT f.release_id, search_normalize(name_stem(coalesce(f.real_name, f.filename))) AS s
        FROM files f WHERE f.release_id IS NOT NULL AND is_readable_name(coalesce(f.real_name, f.filename))) x
      WHERE s <> '' GROUP BY release_id`)
    await q(`CREATE UNIQUE INDEX ON search_stems_tmp (release_id)`)
    let last = 0
    let done = 0
    while (true) {
      const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH])
      if (rows.length === 0) break
      const to = Number(rows[rows.length - 1].id)
      await q(
        `UPDATE releases r SET search_files = (
           SELECT nullif(array_to_string(array(
             SELECT w FROM unnest(string_to_array(t.stems, ' ')) w
             WHERE position(w IN search_normalize(r.name)) = 0), ' '), '')
           FROM search_stems_tmp t WHERE t.release_id = r.id),
           search_text = NULL
         WHERE r.id > ? AND r.id <= ? AND ${VISIBLE.replaceAll('file_count', 'r.file_count').replace('source', 'r.source')}`,
        [last, to]
      )
      last = to
      done += rows.length
      if (done % 200_000 < BATCH) console.log(`Search text for ${done} releases.`)
    }
    await q(`DROP TABLE search_stems_tmp`)

    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_search_trgm ON releases USING gin (search_text gin_trgm_ops) WHERE ${VISIBLE}`)
    for (const index of ['releases_name_trgm', 'releases_contents_trgm', 'files_filename_trgm', 'files_real_name_trgm']) {
      await q(`DROP INDEX CONCURRENTLY IF EXISTS ${index}`)
    }
  }

  async down() {
    const q = (sql: string) => this.db.rawQuery(sql)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops) WHERE ${VISIBLE}`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops) WHERE ${VISIBLE}`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)`)
    await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_search_trgm`)
    await q(`DROP TRIGGER IF EXISTS files_release_search ON files`)
    await q(`DROP TRIGGER IF EXISTS releases_search_text ON releases`)
    await q(`DROP FUNCTION IF EXISTS files_release_search()`)
    await q(`DROP FUNCTION IF EXISTS releases_search_text()`)
    await q(`DROP FUNCTION IF EXISTS release_search_text(text, jsonb, text)`)
    await q(`DROP FUNCTION IF EXISTS search_normalize(text)`)
    await q(`ALTER TABLE releases DROP COLUMN IF EXISTS search_files, DROP COLUMN IF EXISTS search_text`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_size_index ON releases (size) WHERE ${VISIBLE}`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_complete_idx ON releases (complete) WHERE ${VISIBLE}`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_posted_idx ON spots (status, posted_at)`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_article_idx ON spots (status, article_number)`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_created_at_index ON spots (status, created_at)`)
    await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS spots_status_release_id_index ON spots (status, release_id)`)
    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_todo_idx`)
    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_hidden_release_idx`)
    await q(`DROP INDEX CONCURRENTLY IF EXISTS spots_details_todo_idx`)
  }
}