Par2 size matching 100x faster; release bookkeeping once per statement
matchPar2Releases checked every same-size release_files row's set and date (18 s per batch of body-derived binaries); now an index on release_files (release_id, size) drops the repeated sizes first: 0.16 s. File inserts updated their release in four row triggers; one statement trigger now updates each release once per INSERT. Completeness is noted in release_dirty and recomputed by gc:scheduler every 30 minutes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
288a6bab8d
commit
be2112f4ad
9 files changed
+208
-11
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import ReleaseCompleteness from '#services/ReleaseCompleteness'
|
||||
|
||||
/** A release with `n` files (each with a binary of size 100000 + i), named f<i>.bin. */
|
||||
async function release(n: number) {
|
||||
@@ -31,12 +32,24 @@ async function listing(releaseId: number, count: number, offset = 0) {
|
||||
SELECT ?::bigint, md5(i::text), 'f' || i || '.bin', 100000 + i, md5(i::text), 'set' FROM generate_series(1 + ?::int, ?::int + ?::int) i`,
|
||||
[releaseId, offset, count, offset]
|
||||
)
|
||||
// Triggers only note the release; gc:scheduler's sweep recomputes it.
|
||||
await ReleaseCompleteness.sweep()
|
||||
return Boolean((await db.from('releases').where('id', releaseId).first()).complete)
|
||||
}
|
||||
|
||||
test.group('release completeness', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('file inserts keep counts, size, extensions and search text current, once per statement', async ({ assert }) => {
|
||||
const id = await release(3)
|
||||
const row = await db.from('releases').where('id', id).first()
|
||||
assert.equal(Number(row.file_count), 3)
|
||||
assert.deepEqual(row.extensions, ['bin'])
|
||||
// Noted for the completeness sweep once, not per file.
|
||||
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
|
||||
assert.equal(Number(count), 1)
|
||||
})
|
||||
|
||||
test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
|
||||
const id = await release(3)
|
||||
assert.isTrue(await listing(id, 3))
|
||||
@@ -47,6 +60,7 @@ test.group('release completeness', (group) => {
|
||||
|
||||
test('a par2 listing of thousands of files is checked in one go', async ({ assert }) => {
|
||||
const id = await release(3000)
|
||||
await ReleaseCompleteness.sweep()
|
||||
const started = Date.now()
|
||||
assert.isTrue(await listing(id, 3000))
|
||||
// Was cubic: minutes for a few thousand rows.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { test } from '@japa/runner'
|
||||
import ReleaseCompleteness from '#services/ReleaseCompleteness'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
@@ -89,9 +90,12 @@ test.group('Web API', (group) => {
|
||||
response.assertBodyContains({ sources: [] })
|
||||
})
|
||||
|
||||
test('completeness follows files and par2 lists through triggers', async ({ assert }) => {
|
||||
test('completeness follows files and par2 lists (triggers note releases, the sweep recomputes)', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete
|
||||
const complete = async (id: number) => {
|
||||
await ReleaseCompleteness.sweep()
|
||||
return (await db.from('releases').where('id', id).first()).complete
|
||||
}
|
||||
const file = (releaseId: number, name: string) => ({
|
||||
filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
|
||||
groups: '[]', segments: [], created_at: now, updated_at: now,
|
||||
|
||||
Reference in new issue
Block a user