Daan MeijerandClaude Opus 5.5
f8ee7e8662
Creating a partition deadlocked with file inserts 11 times in an hour, and each failure ended the GC sweep before its deletes. The parent is now locked first (lock_timeout 10 s), four partitions are kept ahead, and an error is logged while the sweep goes on. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
51 lines
2.4 KiB
TypeScript
51 lines
2.4 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
|
||
/** Ids per partition (as the migration that partitioned files: …150). */
|
||
export const FILE_PARTITION_SIZE = 4_000_000
|
||
/**
|
||
* Ids kept covered beyond the newest file: four partitions (16M ids, about a day of
|
||
* seeding), so a few sweeps that can't create one are harmless.
|
||
*/
|
||
const AHEAD = 4 * FILE_PARTITION_SIZE
|
||
|
||
/**
|
||
* `files` is partitioned by id (CONCEPT.md D39). An insert with an id no
|
||
* partition covers fails, so gc:scheduler calls ensure() every sweep to keep
|
||
* four partitions ready ahead of the newest id. Each new partition gets its
|
||
* own unique index on file_key (FileStore.insert checks the older ones),
|
||
* in the parent's tablespace (the HDD on homelab).
|
||
*/
|
||
class FilePartitions {
|
||
async ensure(): Promise<string[]> {
|
||
const created: string[] = []
|
||
while (true) {
|
||
// The sequence, not max(id): cheaper, and it counts ids handed out but not committed yet.
|
||
const { rows } = await db.rawQuery(
|
||
`SELECT (SELECT last_value FROM files_id_seq) AS max_id,
|
||
-- ''{0,1}, not ''?: knex reads ? as a binding.
|
||
max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
|
||
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
|
||
WHERE i.inhparent = 'files'::regclass`
|
||
)
|
||
const maxId = Number(rows[0].max_id)
|
||
const upper = Number(rows[0].upper)
|
||
if (upper - maxId >= AHEAD) return created
|
||
const name = `files_p${upper}`
|
||
// CREATE … PARTITION OF locks files and its partitions one by one, which deadlocked with
|
||
// file inserts (11 failed sweeps on 2026-10-07). Taking the parent's lock first, in one
|
||
// statement, can only wait; after 10 s it gives up until the next sweep.
|
||
await db.transaction(async (trx) => {
|
||
await trx.rawQuery(`SET LOCAL lock_timeout = '10s'`)
|
||
await trx.rawQuery(`LOCK TABLE files IN ACCESS EXCLUSIVE MODE`)
|
||
await trx.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
|
||
await trx.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
|
||
})
|
||
logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`)
|
||
created.push(name)
|
||
}
|
||
}
|
||
}
|
||
|
||
export default new FilePartitions()
|