Daan MeijerandClaude Opus 5.5
4d17c5a8e8
release_complete() matched each par2-listed file through a correlated subquery that scanned all binaries per listed file, run twice per inserted release_files row: one insert of a few thousand rows ran for 20+ minutes. It now reads the release's files once by release id, and release_files refreshes once per INSERT statement. seed:node's SEED_CONNECTIONS/--connections is now SEED_RANGES/--ranges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
56 lines
2.5 KiB
TypeScript
56 lines
2.5 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
|
|
/** A release with `n` files (each with a binary of size 100000 + i), named f<i>.bin. */
|
|
async function release(n: number) {
|
|
const now = new Date()
|
|
const [row] = await db
|
|
.table('releases')
|
|
.insert({ release_key: `complete-${n}`.padEnd(40, '0'), name: `Complete.Test.${n}`, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
|
.returning('id')
|
|
const releaseId = Number(row.id)
|
|
await db.rawQuery(
|
|
`WITH f AS (
|
|
INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
|
|
SELECT 'f' || i || '.bin', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'complete-' || ?::int || '-' || i, now(), now()
|
|
FROM generate_series(1, ?::int) i
|
|
RETURNING id, filename)
|
|
INSERT INTO binaries (file_key, poster, subject, name, total_parts, groups, first_seen_at, last_seen_at, size, file_id, release_id)
|
|
SELECT 'complete-b-' || ?::int || '-' || f.id, 'x', 'x', f.filename, 1, '[]', now(), now(), 100000 + substr(f.filename, 2, length(f.filename) - 5)::int, f.id, ?::bigint
|
|
FROM f`,
|
|
[releaseId, n, n, n, releaseId]
|
|
)
|
|
return releaseId
|
|
}
|
|
|
|
/** The par2 listing of `count` files f1..f<count> (sizes as in release()), in one INSERT, as Par2Service.attach does. */
|
|
async function listing(releaseId: number, count: number, offset = 0) {
|
|
await db.rawQuery(
|
|
`INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
|
|
SELECT ?::bigint, md5(i::text), 'f' || i || '.bin', 100000 + i, md5(i::text), 'set' FROM generate_series(1 + ?::int, ?::int + ?::int) i`,
|
|
[releaseId, offset, count, offset]
|
|
)
|
|
return Boolean((await db.from('releases').where('id', releaseId).first()).complete)
|
|
}
|
|
|
|
test.group('release completeness', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
|
|
const id = await release(3)
|
|
assert.isTrue(await listing(id, 3))
|
|
const other = await release(2)
|
|
// f3 is listed but missing.
|
|
assert.isFalse(await listing(other, 3))
|
|
})
|
|
|
|
test('a par2 listing of thousands of files is checked in one go', async ({ assert }) => {
|
|
const id = await release(3000)
|
|
const started = Date.now()
|
|
assert.isTrue(await listing(id, 3000))
|
|
// Was cubic: minutes for a few thousand rows.
|
|
assert.isBelow(Date.now() - started, 5000)
|
|
}).timeout(60_000)
|
|
})
|