import { BaseSchema } from '@adonisjs/lucid/schema' /** * A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)). * file_key is unique per files partition only (D39), so FileStore.insert * looked every key up in every partition: 7 partitions and growing, 35,000 * random index reads for a batch of 5,000 files. Now it asks file_keys * first, and only keys whose hash is there (a duplicate, or rarely a * collision) are looked up in files. * * Filled by the files insert statement trigger; no transaction, so that the * trigger is live before the existing keys are copied and no file inserted * meanwhile is missed (a missed key would let a duplicate file in). */ export default class extends BaseSchema { static disableTransactions = true async up() { const q = (sql: string) => this.db.rawQuery(sql) await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`) await q(` CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN WITH added AS ( SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes, array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts, array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words FROM ( SELECT release_id, size, name_extension(nm) AS e, CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f ) x GROUP BY release_id ) UPDATE releases r SET file_count = r.file_count + a.n, size = r.size + a.bytes, extensions = CASE WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END, search_files = CASE WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files ELSE nullif(left(concat_ws(' ', r.search_files, ( SELECT string_agg(w, ' ') FROM unnest(a.words) w WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END FROM added a WHERE r.id = a.release_id; INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL; INSERT INTO file_keys (key_hash) SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL ON CONFLICT DO NOTHING; RETURN NULL; END $$`) // The keys from before; read from the file_key indexes (NVMe), not the files heap (HDD). await q(`SET enable_seqscan = off`) await q(` INSERT INTO file_keys (key_hash) SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE file_key IS NOT NULL ON CONFLICT DO NOTHING`) await q(`RESET enable_seqscan`) await q(`ANALYZE file_keys`) } async down() { await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`) } }