import { BaseSchema } from '@adonisjs/lucid/schema' const VISIBLE = `(file_count > 0 OR source = 'spot')` const BATCH = 20_000 /** * `releases.extensions`: the file types in a release (`{mkv,nfo,par2}`), from * its files' names and its archive listing, for the search's extension * filter (LOG.md 2026-10-04 (3)). search_text can't serve it: file names * enter it as stems without extension, and matching the long, TOASTed text * took seconds per search. */ export default class extends BaseSchema { static disableTransactions = true async up() { const q = (sql: string, bindings: unknown[] = []) => this.db.rawQuery(sql, bindings as any) await q(`ALTER TABLE releases ADD COLUMN IF NOT EXISTS extensions text[]`) // Two to four characters with a letter (mkv, flac, 7z, par2); split // archive volumes count as their archive type (.7z.001, .r00). await q(` CREATE OR REPLACE FUNCTION name_extension(n text) RETURNS text IMMUTABLE LANGUAGE sql AS $$ SELECT CASE WHEN lower(n) ~ '\\.(7z|zip)\\.[0-9]{3}$' THEN substring(lower(n) from '\\.(7z|zip)\\.[0-9]{3}$') WHEN lower(n) ~ '\\.r[0-9]{2,3}$' THEN 'rar' ELSE substring(lower(n) from '\\.([a-z0-9]*[a-z][a-z0-9]*)$') END $$`) await q(` CREATE OR REPLACE FUNCTION listing_extensions(contents jsonb) RETURNS text[] IMMUTABLE LANGUAGE sql AS $$ SELECT array(SELECT DISTINCT e FROM ( SELECT name_extension(c->>'name') AS e FROM jsonb_array_elements(CASE WHEN jsonb_typeof(contents) = 'array' THEN contents ELSE '[]'::jsonb END) c) x WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 ORDER BY e) $$`) // A new archive listing adds its entries' types. await q(` CREATE OR REPLACE FUNCTION releases_extensions() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF TG_OP = 'INSERT' OR NEW.contents IS DISTINCT FROM OLD.contents THEN NEW.extensions := nullif(array( SELECT DISTINCT e FROM unnest(coalesce(NEW.extensions, '{}') || listing_extensions(NEW.contents)) e ORDER BY e), '{}'); END IF; RETURN NEW; END $$`) await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`) await q(`CREATE TRIGGER releases_extensions BEFORE INSERT OR UPDATE OF contents ON releases FOR EACH ROW EXECUTE FUNCTION releases_extensions()`) // A file adds its type to its release (once: most files of a release share it). await q(` CREATE OR REPLACE FUNCTION files_release_extension() RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE e text := name_extension(coalesce(NEW.real_name, NEW.filename)); BEGIN IF NEW.release_id IS NULL OR e IS NULL OR length(e) NOT BETWEEN 2 AND 4 THEN RETURN NULL; END IF; UPDATE releases SET extensions = array(SELECT DISTINCT x FROM unnest(coalesce(extensions, '{}') || e) x ORDER BY x) WHERE id = NEW.release_id AND NOT (coalesce(extensions, '{}') @> ARRAY[e]); RETURN NULL; END $$`) await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`) await q(`CREATE TRIGGER files_release_extension AFTER INSERT OR UPDATE OF release_id, filename, real_name ON files FOR EACH ROW EXECUTE FUNCTION files_release_extension()`) // Fill: file types per release in one sequential pass over files (random // reads per release would take over an hour on the HDD), then the // visible releases in batches by id, adding their listing's types. await q(`DROP TABLE IF EXISTS file_extensions_tmp`) await q(` CREATE UNLOGGED TABLE file_extensions_tmp AS SELECT release_id, array_agg(DISTINCT e) AS exts FROM ( SELECT release_id, name_extension(coalesce(real_name, filename)) AS e FROM files WHERE release_id IS NOT NULL) x WHERE e IS NOT NULL AND length(e) BETWEEN 2 AND 4 GROUP BY release_id`) await q(`CREATE UNIQUE INDEX ON file_extensions_tmp (release_id)`) let last = 0 let done = 0 while (true) { const { rows } = await q(`SELECT id FROM releases WHERE id > ? AND ${VISIBLE} ORDER BY id LIMIT ?`, [last, BATCH]) if (rows.length === 0) break const to = Number(rows[rows.length - 1].id) await q( `UPDATE releases r SET extensions = x.exts FROM ( SELECT r2.id, array(SELECT DISTINCT e FROM unnest(coalesce(t.exts, '{}') || listing_extensions(r2.contents)) e ORDER BY e) AS exts FROM releases r2 LEFT JOIN file_extensions_tmp t ON t.release_id = r2.id WHERE r2.id > ? AND r2.id <= ? AND (r2.file_count > 0 OR r2.source = 'spot')) x WHERE r.id = x.id AND cardinality(x.exts) > 0 AND r.extensions IS DISTINCT FROM x.exts`, [last, to] ) last = to done += rows.length if (done % 500_000 < BATCH) console.log(`Extensions for ${done} releases.`) } await q(`DROP TABLE file_extensions_tmp`) await q(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_extensions_gin ON releases USING gin (extensions) WHERE ${VISIBLE}`) } async down() { const q = (sql: string) => this.db.rawQuery(sql) await q(`DROP INDEX CONCURRENTLY IF EXISTS releases_extensions_gin`) await q(`DROP TRIGGER IF EXISTS files_release_extension ON files`) await q(`DROP TRIGGER IF EXISTS releases_extensions ON releases`) await q(`DROP FUNCTION IF EXISTS files_release_extension()`) await q(`DROP FUNCTION IF EXISTS releases_extensions()`) await q(`DROP FUNCTION IF EXISTS listing_extensions(jsonb)`) await q(`DROP FUNCTION IF EXISTS name_extension(text)`) await q(`ALTER TABLE releases DROP COLUMN IF EXISTS extensions`) } }