import { BaseSchema } from '@adonisjs/lucid/schema' /** * A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)). * file_key is unique per files partition only (D39), so FileStore.insert * looked every key up in every partition: 7 partitions and growing, 35,000 * random index reads for a batch of 5,000 files. With file_keys complete it * asks file_keys first, and only keys whose hash is there (a duplicate, or * rarely a collision) are looked up in files. * * New keys come from the files insert statement trigger. The keys from * before are copied in the background (FileKeys.fill, gc:scheduler): * copying them here took over half an hour of HDD reads and held up the * deploy. Until file_keys_fill.complete, FileStore checks every partition. */ export default class extends BaseSchema { async up() { const q = (sql: string) => this.db.rawQuery(sql) await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`) // One row: the fill's position in files.id, the id it must reach, and whether it got there. await q(` CREATE TABLE IF NOT EXISTS file_keys_fill ( id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1), next_id bigint NOT NULL DEFAULT 0, until_id bigint, complete boolean NOT NULL DEFAULT false, updated_at timestamptz NOT NULL DEFAULT now())`) await q(`INSERT INTO file_keys_fill (id) VALUES (1) ON CONFLICT DO NOTHING`) await q(` CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN WITH added AS ( SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes, array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts, array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words FROM ( SELECT release_id, size, name_extension(nm) AS e, CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f ) x GROUP BY release_id ) UPDATE releases r SET file_count = r.file_count + a.n, size = r.size + a.bytes, extensions = CASE WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END, search_files = CASE WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files ELSE nullif(left(concat_ws(' ', r.search_files, ( SELECT string_agg(w, ' ') FROM unnest(a.words) w WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END FROM added a WHERE r.id = a.release_id; INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL; INSERT INTO file_keys (key_hash) SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL ON CONFLICT DO NOTHING; RETURN NULL; END $$`) } async down() { await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys_fill`) await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`) } }