Daan MeijerandClaude Opus 5.5
693e6c3244
- seed:plan plans every part of a group's history no range covers (it started below the lowest known range and left a.b.ath's 13 most recent months out); seed_ranges.priority orders leases (seed:plan --priority). - binaries.part_count, kept by the parts insert in the same statement: claimCompleted compares it instead of counting parts every batch. - file_keys: 64-bit hashes of all file keys; FileStore.insert asks it before probing every files partition. - worker:file stores 100 completed binaries per job with set-based queries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69 lines
3.2 KiB
TypeScript
69 lines
3.2 KiB
TypeScript
import { BaseSchema } from '@adonisjs/lucid/schema'
|
|
|
|
/**
|
|
* A 64-bit hash of every file_key, in one index (LOG.md 2026-10-06 (7)).
|
|
* file_key is unique per files partition only (D39), so FileStore.insert
|
|
* looked every key up in every partition: 7 partitions and growing, 35,000
|
|
* random index reads for a batch of 5,000 files. Now it asks file_keys
|
|
* first, and only keys whose hash is there (a duplicate, or rarely a
|
|
* collision) are looked up in files.
|
|
*
|
|
* Filled by the files insert statement trigger; no transaction, so that the
|
|
* trigger is live before the existing keys are copied and no file inserted
|
|
* meanwhile is missed (a missed key would let a duplicate file in).
|
|
*/
|
|
export default class extends BaseSchema {
|
|
static disableTransactions = true
|
|
|
|
async up() {
|
|
const q = (sql: string) => this.db.rawQuery(sql)
|
|
await q(`CREATE TABLE IF NOT EXISTS file_keys (key_hash bigint PRIMARY KEY)`)
|
|
await q(`
|
|
CREATE OR REPLACE FUNCTION files_release_insert_statement() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
WITH added AS (
|
|
SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS bytes,
|
|
array_agg(DISTINCT e) FILTER (WHERE length(e) BETWEEN 2 AND 4) AS exts,
|
|
array_agg(DISTINCT w) FILTER (WHERE w <> '') AS words
|
|
FROM (
|
|
SELECT release_id, size, name_extension(nm) AS e,
|
|
CASE WHEN is_readable_name(nm) THEN search_normalize(name_stem(nm)) END AS w
|
|
FROM (SELECT release_id, size, coalesce(real_name, filename) AS nm FROM new_rows WHERE release_id IS NOT NULL) f
|
|
) x
|
|
GROUP BY release_id
|
|
)
|
|
UPDATE releases r SET
|
|
file_count = r.file_count + a.n,
|
|
size = r.size + a.bytes,
|
|
extensions = CASE
|
|
WHEN a.exts IS NULL OR coalesce(r.extensions, '{}') @> a.exts THEN r.extensions
|
|
ELSE array(SELECT DISTINCT x FROM unnest(coalesce(r.extensions, '{}') || a.exts) x ORDER BY x) END,
|
|
search_files = CASE
|
|
WHEN a.words IS NULL OR length(coalesce(r.search_files, '')) >= 4000 THEN r.search_files
|
|
ELSE nullif(left(concat_ws(' ', r.search_files, (
|
|
SELECT string_agg(w, ' ') FROM unnest(a.words) w
|
|
WHERE position(w IN coalesce(r.search_text, search_normalize(r.name))) = 0)), 4000), '') END
|
|
FROM added a
|
|
WHERE r.id = a.release_id;
|
|
|
|
INSERT INTO release_dirty (release_id) SELECT DISTINCT release_id FROM new_rows WHERE release_id IS NOT NULL;
|
|
INSERT INTO file_keys (key_hash)
|
|
SELECT DISTINCT hashtextextended(file_key, 0) FROM new_rows WHERE file_key IS NOT NULL
|
|
ON CONFLICT DO NOTHING;
|
|
RETURN NULL;
|
|
END $$`)
|
|
// The keys from before; read from the file_key indexes (NVMe), not the files heap (HDD).
|
|
await q(`SET enable_seqscan = off`)
|
|
await q(`
|
|
INSERT INTO file_keys (key_hash)
|
|
SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE file_key IS NOT NULL
|
|
ON CONFLICT DO NOTHING`)
|
|
await q(`RESET enable_seqscan`)
|
|
await q(`ANALYZE file_keys`)
|
|
}
|
|
|
|
async down() {
|
|
await this.db.rawQuery(`DROP TABLE IF EXISTS file_keys`)
|
|
}
|
|
}
|