import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import PendingLadder, { PENDING, byEffort, type PendingProbe } from '#services/pending_ladder' import { identityKey, releaseKey } from '#lib/release/keys' let groupId = 0 let n = 0 async function pending( name: string, candidates: { kind: string; messageId: string; file: string; bytes?: number }[], extra: Record = {} ) { n++ const key = releaseKey(name, groupId, n) await db.table('pending_releases').insert({ release_key: key, identity_key: identityKey(name), group_id: groupId, name, poster: 'Bob ', first_article: n, last_article: n, files: 1, segments: 10, bytes: 1000, tool: 'other', candidates: JSON.stringify(candidates), ...extra, }) return key.toString('hex') } const inspection = (yencName: string, names?: string[]) => ({ yencName, size: 1000, type: null, names, entries: names?.length, }) test.group('Pending ladder', (group) => { group.each.setup(async () => { const rollback = await testUtils.db().withGlobalTransaction() groupId = Number( ( await db .table('groups') .insert({ name: 'alt.binaries.pending', priority: 1 }) .returning('id') )[0].id ) n = 0 return rollback }) test('targets are climbed by effort: a par2 before an archive before a lone file', ({ assert, }) => { assert.deepEqual( byEffort([ { kind: 'lone', messageId: 'l', file: 'l' }, { kind: '7z-tail', messageId: 't', file: 't' }, { kind: 'archive', messageId: 'a', file: 'a' }, { kind: 'par2', messageId: 'p', file: 'p' }, ]).map((t) => t.kind), ['par2', 'archive', 'lone', '7z-tail'] ) }) test('a claim takes the oldest waiting releases, their cheapest target first; outcomes move them up or to the end', async ({ assert, }) => { const twoTargets = await pending('a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4', [ { kind: 'archive', messageId: '', file: 'x.rar' }, { kind: 'par2', messageId: '', file: 'x.par2' }, ]) const lone = await pending('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3', [ { kind: 'lone', messageId: '', file: 'f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3' }, ]) const none = await pending('00000000000000000000000000000001', []) const provisional = await pending( '00000000000000000000000000000002', [{ kind: 'lone', messageId: '', file: 'v' }], { finished: false } ) const probes = await PendingLadder.claim(10) assert.deepEqual( probes.map((p) => [p.releaseKey, p.kind, p.tries, p.targets]), [ [twoTargets, 'par2', 0, 2], [lone, 'lone', 0, 1], ] ) const statusOf = async (key: string) => (await db .from('pending_releases') .where('release_key', Buffer.from(key, 'hex')) .firstOrFail()) as any assert.equal((await statusOf(twoTargets)).status, PENDING.taken) assert.equal((await statusOf(none)).status, PENDING.waiting) assert.equal((await statusOf(provisional)).status, PENDING.waiting) // The par2 names only random volumes: the next target. The lone file's yEnc name is its random name: exhausted. await PendingLadder.record([ { probe: probes[0], status: 'done', inspection: inspection('x.par2', ['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4.part01.rar']), }, { probe: probes[1], status: 'done', inspection: inspection('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3'), }, ]) const first = await statusOf(twoTargets) assert.include(first, { status: PENDING.waiting, tries: 1, named: null }) assert.lengthOf(first.inspections, 1) assert.include(await statusOf(lone), { status: PENDING.exhausted, tries: 1 }) // The archive's listing names it: for the namer. const [second] = await PendingLadder.claim(10) assert.deepEqual([second.releaseKey, second.kind, second.tries], [twoTargets, 'archive', 1]) await PendingLadder.record([ { probe: second, status: 'done', inspection: inspection('x.rar', ['Some.Movie.2016.1080p.mkv']), }, ]) assert.include(await statusOf(twoTargets), { status: PENDING.naming, named: 'Some.Movie.2016.1080p.mkv', tries: 0, }) // The signature counted both ways: the poster's own and the tool's in the group. const signatures = await db.from('pending_signatures').orderBy('poster') assert.deepEqual( signatures.map((s) => [s.tool, s.poster, s.probed, s.named, s.dropped]), [ ['other', '', 2, 1, false], ['other', 'Bob ', 2, 1, false], ] ) }) test('a signature that names nothing is dropped, and its waiting releases discarded without a probe', async ({ assert, }) => { // 100 climbs of the tool in the group, none named. await db .table('pending_signatures') .insert({ tool: 'other', group_id: groupId, poster: '', probed: 99, named: 0 }) const last = await pending('00000000000000000000000000000009', [ { kind: 'lone', messageId: '', file: 'z' }, ]) const [probe] = await PendingLadder.claim(1) assert.equal(probe.releaseKey, last) await PendingLadder.record([{ probe, status: 'missing' }]) const signature = await db .from('pending_signatures') .where({ tool: 'other', group_id: groupId, poster: '' }) .firstOrFail() assert.include(signature, { probed: 100, named: 0, dropped: true }) const waiting = await pending('00000000000000000000000000000010', [ { kind: 'lone', messageId: '', file: 'w' }, ]) const other = await pending( '00000000000000000000000000000011', [{ kind: 'lone', messageId: '', file: 'o' }], { tool: 'nyuu' } ) // samplePercent 0 in the test rules: no explored sample. await db.table('rule_sets').insert({ version: 99, rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }), note: 'test', }) assert.equal(await PendingLadder.discard(), 1) const statusOf = async (key: string) => ( await db .from('pending_releases') .where('release_key', Buffer.from(key, 'hex')) .firstOrFail() ).status assert.equal(await statusOf(waiting), PENDING.discarded) assert.equal(await statusOf(other), PENDING.waiting) }) test('a probe given back goes to the queue again at the same step', async ({ assert }) => { const key = await pending('00000000000000000000000000000020', [ { kind: 'lone', messageId: '', file: 'g' }, ]) const [probe] = await PendingLadder.claim(1) await PendingLadder.record([{ probe: probe as PendingProbe, status: 'waiting' }]) assert.include( await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(), { status: PENDING.waiting, tries: 0 } ) assert.lengthOf(await db.from('pending_signatures'), 0) }) }) test.group('Pending ladder / classes', (group) => { group.each.setup(async () => { PendingLadder.forget() const rollback = await testUtils.db().withGlobalTransaction() groupId = Number( ( await db .table('groups') .insert({ name: 'alt.binaries.classes', priority: 1 }) .returning('id') )[0].id ) n = 0 return rollback }) test('a claim takes the class with the best record first, the cheapest target within it, the rest oldest first', async ({ assert, }) => { // Class 15 (1-10 MB hex, whole) named 9 of 10; class 16 (partial) named 0 of 50; class 0: rows from before. await db.table('pending_classes').multiInsert([ { tool: 'other', group_id: groupId, klass: 15, probed: 10, named: 9, waiting: 2 }, { tool: 'other', group_id: groupId, klass: 16, probed: 50, named: 0, waiting: 1 }, ]) const old = await pending('00000000000000000000000000000031', [ { kind: 'lone', messageId: '', file: 'o' }, ]) const poor = await pending( '00000000000000000000000000000032', [{ kind: 'lone', messageId: '', file: 'p' }], { klass: 16 } ) const bigTarget = await pending( '00000000000000000000000000000033', [{ kind: 'archive', messageId: '', file: 'b.rar', bytes: 700_000 }], { klass: 15, target_bytes: 700_000 } ) const smallTarget = await pending( '00000000000000000000000000000034', [{ kind: 'archive', messageId: '', file: 's.rar', bytes: 50_000 }], { klass: 15, target_bytes: 50_000 } ) const probes = await PendingLadder.claim(10) assert.deepEqual( probes.map((p) => [p.releaseKey, p.klass]), [ [smallTarget, 15], [bigTarget, 15], [poor, 16], [old, 0], ] ) const waiting = await db .from('pending_classes') .where({ group_id: groupId }) .orderBy('klass') .select('klass', 'waiting') assert.deepEqual( waiting.map((c) => [c.klass, c.waiting]), [ [15, 0], [16, 0], ] ) // Outcomes count on the class; a probe given back waits again. await PendingLadder.record([ { probe: probes[0], status: 'done', inspection: inspection('s.rar', ['Some.Movie.2016.1080p.mkv']), }, { probe: probes[1], status: 'waiting' }, { probe: probes[2], status: 'missing' }, ]) const classes = await db.from('pending_classes').where({ group_id: groupId }).orderBy('klass') assert.deepEqual( classes.map((c) => [c.klass, c.probed, c.named, c.waiting, c.dropped]), [ [15, 11, 10, 1, false], [16, 51, 0, 0, false], ] ) }) test('a class that names nothing is dropped and its waiting releases discarded, the other classes untouched', async ({ assert, }) => { await db .table('pending_classes') .insert({ tool: 'other', group_id: groupId, klass: 16, probed: 99, named: 0, waiting: 1 }) const last = await pending( '00000000000000000000000000000041', [{ kind: 'lone', messageId: '', file: 'z' }], { klass: 16 } ) const [probe] = await PendingLadder.claim(1) assert.equal(probe.releaseKey, last) await PendingLadder.record([{ probe, status: 'missing' }]) assert.include( await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail(), { probed: 100, named: 0, dropped: true } ) const waiting = await pending( '00000000000000000000000000000042', [{ kind: 'lone', messageId: '', file: 'w' }], { klass: 16 } ) const other = await pending( '00000000000000000000000000000043', [{ kind: 'lone', messageId: '', file: 'o' }], { klass: 15 } ) await db.rawQuery( `UPDATE pending_classes SET waiting = waiting + 1 WHERE group_id = ? AND klass = 16`, [groupId] ) await db.table('rule_sets').insert({ version: 98, rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }), note: 'test', }) assert.equal(await PendingLadder.discard(), 1) const statusOf = async (key: string) => ( await db .from('pending_releases') .where('release_key', Buffer.from(key, 'hex')) .firstOrFail() ).status assert.equal(await statusOf(waiting), PENDING.discarded) assert.equal(await statusOf(other), PENDING.waiting) assert.equal( (await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail()) .waiting, 0 ) }) test('a twin by content names a release whose own probe did not', async ({ assert }) => { // A catalog release with a readable name holds the same first 16 KiB and size. const [poster] = await db.table('posters').insert({ poster: 'Twin ' }).returning('id') const [set] = await db .table('group_sets') .insert({ groups: JSON.stringify(['alt.binaries.classes']) }) .returning('id') const [twin] = await db .table('releases') .insert({ release_key: Buffer.from('aa'.repeat(20), 'hex'), identity_key: Buffer.from('bb'.repeat(20), 'hex'), group_id: groupId, group_set_id: set.id, poster_id: poster.id, name: 'Some.Movie.2016.1080p.BluRay-GRP', source: 3, posted_at: new Date('2026-10-01T00:00:00Z'), first_article: 1, last_article: 2, files: 2, segments: 2, bytes: 1000, readable: true, }) .returning('id') await db.table('file_hashes').insert({ candidate_id: 777_001, release_id: twin.id, hash16k: 'deadbeef'.repeat(4), size: 1000, }) const key = await pending('00000000000000000000000000000051', [ { kind: 'lone', messageId: '', file: '00000000000000000000000000000051' }, ]) const [probe] = await PendingLadder.claim(1) assert.equal(probe.releaseKey, key) await PendingLadder.record([ { probe, status: 'done', inspection: { ...inspection('00000000000000000000000000000051'), hash16k: 'deadbeef'.repeat(4), }, }, ]) assert.include( await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(), { status: PENDING.naming, named: 'Some.Movie.2016.1080p.BluRay-GRP', } ) }) })