Par2 size matching 100x faster; release bookkeeping once per statement
Test and deploy / test (push) Successful in 1m33s
Test and deploy / deploy (push) Failing after 1m29s

matchPar2Releases checked every same-size release_files row's set and
date (18 s per batch of body-derived binaries); now an index on
release_files (release_id, size) drops the repeated sizes first: 0.16 s.

File inserts updated their release in four row triggers; one statement
trigger now updates each release once per INSERT. Completeness is noted
in release_dirty and recomputed by gc:scheduler every 30 minutes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-06 13:48:59 +02:00
1 parent 288a6bab8d
commit be2112f4ad
9 files changed
+208 -11

No files matched your search

+14
View File
@@ -1,6 +1,7 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
/** A release with `n` files (each with a binary of size 100000 + i), named f<i>.bin. */
async function release(n: number) {
@@ -31,12 +32,24 @@ async function listing(releaseId: number, count: number, offset = 0) {
SELECT ?::bigint, md5(i::text), 'f' || i || '.bin', 100000 + i, md5(i::text), 'set' FROM generate_series(1 + ?::int, ?::int + ?::int) i`,
[releaseId, offset, count, offset]
)
// Triggers only note the release; gc:scheduler's sweep recomputes it.
await ReleaseCompleteness.sweep()
return Boolean((await db.from('releases').where('id', releaseId).first()).complete)
}
test.group('release completeness', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('file inserts keep counts, size, extensions and search text current, once per statement', async ({ assert }) => {
const id = await release(3)
const row = await db.from('releases').where('id', id).first()
assert.equal(Number(row.file_count), 3)
assert.deepEqual(row.extensions, ['bin'])
// Noted for the completeness sweep once, not per file.
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
assert.equal(Number(count), 1)
})
test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
const id = await release(3)
assert.isTrue(await listing(id, 3))
@@ -47,6 +60,7 @@ test.group('release completeness', (group) => {
test('a par2 listing of thousands of files is checked in one go', async ({ assert }) => {
const id = await release(3000)
await ReleaseCompleteness.sweep()
const started = Date.now()
assert.isTrue(await listing(id, 3000))
// Was cubic: minutes for a few thousand rows.
+6 -2
View File
@@ -1,4 +1,5 @@
import { test } from '@japa/runner'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import testUtils from '@adonisjs/core/services/test_utils'
@@ -89,9 +90,12 @@ test.group('Web API', (group) => {
response.assertBodyContains({ sources: [] })
})
test('completeness follows files and par2 lists through triggers', async ({ assert }) => {
test('completeness follows files and par2 lists (triggers note releases, the sweep recomputes)', async ({ assert }) => {
const now = new Date()
const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete
const complete = async (id: number) => {
await ReleaseCompleteness.sweep()
return (await db.from('releases').where('id', id).first()).complete
}
const file = (releaseId: number, name: string) => ({
filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
groups: '[]', segments: [], created_at: now, updated_at: now,