Daan MeijerandClaude Opus 5.5
f0ff8e200f
Copying all existing keys in the migration read the files rows from the HDD for over half an hour and held up the deploy. gc:scheduler now copies them in slices (FILE_KEYS_FILL_SECONDS per sweep) and FileStore checks every partition until the fill is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
70 lines
3.4 KiB
TypeScript
70 lines
3.4 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
|
|
/**
|
|
* A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)).
|
|
* file_key is unique per files partition only (D39), so FileStore.insert
|
|
* looked every key up in every partition: 7 partitions and growing, 35,000
|
|
* random index reads for a batch of 5,000 files. With file_keys complete it
|
|
* asks file_keys first, and only keys whose hash is there (a duplicate, or
|
|
* rarely a collision) are looked up in files.
|
|
*
|
|
* New keys come from the files insert statement trigger. The keys from
|
|
* before are copied in the background (FileKeys.fill, gc:scheduler):
|
|
* copying them here took over half an hour of HDD reads and held up the
|
|
* deploy. Until file_keys_fill.complete, FileStore checks every partition.
|
|
*/
|
|
export default class extends BaseSchema {
|
|
async up() {
|
|
const q = (sql: string) => this.db.rawQuery(sql)
|
|
await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`)
|
|
// One row: the fill's position in files.id, the id it must reach, and whether it got there.
|
|
await q(`
|
|
CREATE TABLE IF NOT EXISTS file_keys_fill (
|
|
id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
|
next_id bigint NOT NULL DEFAULT 0,
|
|
until_id bigint,
|
|
complete boolean NOT NULL DEFAULT false,
|
|
updated_at timestamptz NOT NULL DEFAULT now())`)
|
|
await q(`INSERT INTO file_keys_fill (id) VALUES (1) ON CONFLICT DO NOTHING`)
|
|
await q(`
|
|
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
WITH added AS (
|
|
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
|
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
|
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
|
FROM (
|
|
SELECT release_id, size, name_extension(nm) AS e,
|
|
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
|
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
|
) x
|
|
GROUP BY release_id
|
|
)
|
|
UPDATE releases r SET
|
|
file_count = r.file_count + a.n,
|
|
size = r.size + a.bytes,
|
|
extensions = CASE
|
|
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
|
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
|
search_files = CASE
|
|
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
|
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
|
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
|
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
|
FROM added a
|
|
WHERE r.id = a.release_id;
|
|
|
|
INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
|
INSERT INTO file_keys (key_hash)
|
|
SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
|
|
ON CONFLICT DO NOTHING;
|
|
RETURN NULL;
|
|
END $$`)
|
|
}
|
|
|
|
async down() {
|
|
await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys_fill`)
|
|
await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`)
|
|
}
|
|
}
|