Daan MeijerandClaude Opus 5.5
f1773744c0
New rows went into free space scattered through the 9.6 GB heap on the HDD (left behind by inspection's updates): one random HDD read per insert, and seed batches took a median 13 s to store. The existing table becomes the first partition as it is (a CHECK scan, no rewrite; indexes, its foreign key and the triggers are taken over by the partitioned parent); new partitions of 4M ids go in the same tablespace and stay in memory while they fill. - Every partition has a unique file_key index; FileStore.insert looks keys up in all partitions first (a key can't be unique across partitions). - FilePartitions.ensure() (gc:scheduler) keeps two partitions ahead. - Stats count files from the partitions; seed:node waits ~20 minutes for the API before giving a range up. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
51 lines
2.6 KiB
TypeScript
51 lines
2.6 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import FileStore from '#services/FileStore'
|
|
import FilePartitions, { FILE_PARTITION_SIZE } from '#services/FilePartitions'
|
|
|
|
const file = (fileKey: string, name: string) => {
|
|
const now = new Date()
|
|
return { file_key: fileKey, filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], created_at: now, updated_at: now }
|
|
}
|
|
|
|
test.group('files partitions', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
// Sequences ignore rollbacks: put files_id_seq back after each test.
|
|
group.each.setup(async () => {
|
|
const { rows } = await db.rawQuery(`SELECT last_value, is_called FROM files_id_seq`)
|
|
return async () => {
|
|
await db.rawQuery(`SELECT setval('files_id_seq', ?, ?)`, [rows[0].last_value, rows[0].is_called])
|
|
}
|
|
})
|
|
|
|
test('a file_key already in an older partition is not inserted again', async ({ assert }) => {
|
|
await db.rawQuery(`SELECT setval('files_id_seq', 500000, false)`)
|
|
const [first] = await FileStore.insert([file('k'.repeat(40), 'old.bin')])
|
|
// The next ids fall in the second partition, whose own unique index doesn't know the key.
|
|
await db.rawQuery(`SELECT setval('files_id_seq', 1000000, false)`)
|
|
const again = await FileStore.insert([file('k'.repeat(40), 'again.bin'), file('n'.repeat(40), 'new.bin'), file('n'.repeat(40), 'twice.bin')])
|
|
assert.deepEqual(again.map((f) => f.filename), ['new.bin'])
|
|
assert.isAtLeast(again[0].id, 1000000)
|
|
assert.equal(await db.from('files').where('file_key', 'k'.repeat(40)).count('* as n').then((r) => Number(r[0].n)), 1)
|
|
assert.isBelow(first.id, 1000000)
|
|
})
|
|
|
|
test('partitions are kept ready ahead of the newest id', async ({ assert }) => {
|
|
const upper = async () => {
|
|
const { rows } = await db.rawQuery(
|
|
`SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
|
|
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass`
|
|
)
|
|
return Number(rows[0].upper)
|
|
}
|
|
const before = await upper()
|
|
assert.deepEqual(await FilePartitions.ensure(), [])
|
|
// A file near the end of the covered ids: one more partition is needed.
|
|
await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - FILE_PARTITION_SIZE - 10])
|
|
await FileStore.insert([file('e'.repeat(40), 'late.bin')])
|
|
assert.deepEqual(await FilePartitions.ensure(), [`files_p${before}`])
|
|
assert.equal(await upper(), before + FILE_PARTITION_SIZE)
|
|
})
|
|
})
|