diff --git a/app/services/FilePartitions.ts b/app/services/FilePartitions.ts index 83e7ece..2915cac 100644 --- a/app/services/FilePartitions.ts +++ b/app/services/FilePartitions.ts @@ -3,13 +3,16 @@ import logger from '@adonisjs/core/services/logger' /** Ids per partition (as the migration that partitioned files: …150). */ export const FILE_PARTITION_SIZE = 4_000_000 -/** Ids kept covered beyond the newest file: two partitions. */ -const AHEAD = 2 * FILE_PARTITION_SIZE +/** + * Ids kept covered beyond the newest file: four partitions (16M ids, about a day of + * seeding), so a few sweeps that can't create one are harmless. + */ +const AHEAD = 4 * FILE_PARTITION_SIZE /** * `files` is partitioned by id (CONCEPT.md D39). An insert with an id no * partition covers fails, so gc:scheduler calls ensure() every sweep to keep - * two partitions ready ahead of the newest id. Each new partition gets its + * four partitions ready ahead of the newest id. Each new partition gets its * own unique index on file_key (FileStore.insert checks the older ones), * in the parent's tablespace (the HDD on homelab). */ @@ -17,8 +20,9 @@ class FilePartitions { async ensure(): Promise { const created: string[] = [] while (true) { + // The sequence, not max(id): cheaper, and it counts ids handed out but not committed yet. const { rows } = await db.rawQuery( - `SELECT (SELECT coalesce(max(id), 0) FROM files) AS max_id, + `SELECT (SELECT last_value FROM files_id_seq) AS max_id, -- ''{0,1}, not ''?: knex reads ? as a binding. max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid @@ -28,8 +32,15 @@ class FilePartitions { const upper = Number(rows[0].upper) if (upper - maxId >= AHEAD) return created const name = `files_p${upper}` - await db.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`) - await db.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`) + // CREATE … PARTITION OF locks files and its partitions one by one, which deadlocked with + // file inserts (11 failed sweeps on 2026-10-07). Taking the parent's lock first, in one + // statement, can only wait; after 10 s it gives up until the next sweep. + await db.transaction(async (trx) => { + await trx.rawQuery(`SET LOCAL lock_timeout = '10s'`) + await trx.rawQuery(`LOCK TABLE files IN ACCESS EXCLUSIVE MODE`) + await trx.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`) + await trx.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`) + }) logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`) created.push(name) } diff --git a/commands/GcScheduler.ts b/commands/GcScheduler.ts index 0dca953..5b45166 100644 --- a/commands/GcScheduler.ts +++ b/commands/GcScheduler.ts @@ -79,8 +79,15 @@ export default class GcScheduler extends BaseCommand { } } - // Inserts fail without a partition for their id (D39). - if (!this.dryRun) await FilePartitions.ensure() + // Inserts fail without a partition for their id (D39). Four are kept ahead, so a failure here + // can wait for the next sweep; it used to end the sweep, deletes and all (LOG.md 2026-10-07 (4)). + if (!this.dryRun) { + try { + await FilePartitions.ensure() + } catch (err: any) { + this.logger.error(`files partitions: ${err.message}; trying again next sweep.`) + } + } const trimmed = await this.trimBodyQueue() if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`) diff --git a/tests/functional/file_partitions.spec.ts b/tests/functional/file_partitions.spec.ts index 2110093..4e8cfae 100644 --- a/tests/functional/file_partitions.spec.ts +++ b/tests/functional/file_partitions.spec.ts @@ -39,10 +39,11 @@ test.group('files partitions', (group) => { ) return Number(rows[0].upper) } + await FilePartitions.ensure() const before = await upper() assert.deepEqual(await FilePartitions.ensure(), []) - // A file near the end of the covered ids: one more partition is needed. - await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - FILE_PARTITION_SIZE - 10]) + // Ids handed out up to just inside the four partitions kept ahead: one more is needed. + await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - 4 * FILE_PARTITION_SIZE + 10]) await FileStore.insert([file('e'.repeat(40), 'late.bin')]) assert.deepEqual(await FilePartitions.ensure(), [`files_p${before}`]) assert.equal(await upper(), before + FILE_PARTITION_SIZE)