usenet-indexer/tests/functional/pending_ladder.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import PendingLadder, { PENDING, byEffort, type PendingProbe } from '#services/pending_ladder'
import { identityKey, releaseKey } from '#lib/release/keys'

let groupId = 0
let n = 0

async function pending(
  name: string,
  candidates: { kind: string; messageId: string; file: string }[],
  extra: Record<string, unknown> = {}
) {
  n++
  const key = releaseKey(name, groupId, n)
  await db.table('pending_releases').insert({
    release_key: key,
    identity_key: identityKey(name),
    group_id: groupId,
    name,
    poster: 'Bob <bob@home.me>',
    first_article: n,
    last_article: n,
    files: 1,
    segments: 10,
    bytes: 1000,
    tool: 'other',
    candidates: JSON.stringify(candidates),
    ...extra,
  })
  return key.toString('hex')
}

const inspection = (yencName: string, names?: string[]) => ({
  yencName,
  size: 1000,
  type: null,
  names,
  entries: names?.length,
})

test.group('Pending ladder', (group) => {
  group.each.setup(async () => {
    const rollback = await testUtils.db().withGlobalTransaction()
    groupId = Number(
      (
        await db
          .table('groups')
          .insert({ name: 'alt.binaries.pending', priority: 1 })
          .returning('id')
      )[0].id
    )
    n = 0
    return rollback
  })

  test('targets are climbed by effort: a par2 before an archive before a lone file', ({
    assert,
  }) => {
    assert.deepEqual(
      byEffort([
        { kind: 'lone', messageId: 'l', file: 'l' },
        { kind: '7z-tail', messageId: 't', file: 't' },
        { kind: 'archive', messageId: 'a', file: 'a' },
        { kind: 'par2', messageId: 'p', file: 'p' },
      ]).map((t) => t.kind),
      ['par2', 'archive', 'lone', '7z-tail']
    )
  })

  test('a claim takes the oldest waiting releases, their cheapest target first; outcomes move them up or to the end', async ({
    assert,
  }) => {
    const twoTargets = await pending('a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4', [
      { kind: 'archive', messageId: '<a@x>', file: 'x.rar' },
      { kind: 'par2', messageId: '<p@x>', file: 'x.par2' },
    ])
    const lone = await pending('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3', [
      { kind: 'lone', messageId: '<l@x>', file: 'f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3' },
    ])
    const none = await pending('00000000000000000000000000000001', [])
    const provisional = await pending(
      '00000000000000000000000000000002',
      [{ kind: 'lone', messageId: '<v@x>', file: 'v' }],
      { finished: false }
    )

    const probes = await PendingLadder.claim(10)
    assert.deepEqual(
      probes.map((p) => [p.releaseKey, p.kind, p.tries, p.targets]),
      [
        [twoTargets, 'par2', 0, 2],
        [lone, 'lone', 0, 1],
      ]
    )
    const statusOf = async (key: string) =>
      (await db
        .from('pending_releases')
        .where('release_key', Buffer.from(key, 'hex'))
        .firstOrFail()) as any
    assert.equal((await statusOf(twoTargets)).status, PENDING.taken)
    assert.equal((await statusOf(none)).status, PENDING.waiting)
    assert.equal((await statusOf(provisional)).status, PENDING.waiting)

    // The par2 names only random volumes: the next target. The lone file's yEnc name is its random name: exhausted.
    await PendingLadder.record([
      {
        probe: probes[0],
        status: 'done',
        inspection: inspection('x.par2', ['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4.part01.rar']),
      },
      {
        probe: probes[1],
        status: 'done',
        inspection: inspection('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3'),
      },
    ])
    const first = await statusOf(twoTargets)
    assert.include(first, { status: PENDING.waiting, tries: 1, named: null })
    assert.lengthOf(first.inspections, 1)
    assert.include(await statusOf(lone), { status: PENDING.exhausted, tries: 1 })

    // The archive's listing names it: for the namer.
    const [second] = await PendingLadder.claim(10)
    assert.deepEqual([second.releaseKey, second.kind, second.tries], [twoTargets, 'archive', 1])
    await PendingLadder.record([
      {
        probe: second,
        status: 'done',
        inspection: inspection('x.rar', ['Some.Movie.2016.1080p.mkv']),
      },
    ])
    assert.include(await statusOf(twoTargets), {
      status: PENDING.naming,
      named: 'Some.Movie.2016.1080p.mkv',
      tries: 0,
    })

    // The signature counted both ways: the poster's own and the tool's in the group.
    const signatures = await db.from('pending_signatures').orderBy('poster')
    assert.deepEqual(
      signatures.map((s) => [s.tool, s.poster, s.probed, s.named, s.dropped]),
      [
        ['other', '', 2, 1, false],
        ['other', 'Bob <bob@home.me>', 2, 1, false],
      ]
    )
  })

  test('a signature that names nothing is dropped, and its waiting releases discarded without a probe', async ({
    assert,
  }) => {
    // 100 climbs of the tool in the group, none named.
    await db
      .table('pending_signatures')
      .insert({ tool: 'other', group_id: groupId, poster: '', probed: 99, named: 0 })
    const last = await pending('00000000000000000000000000000009', [
      { kind: 'lone', messageId: '<z@x>', file: 'z' },
    ])
    const [probe] = await PendingLadder.claim(1)
    assert.equal(probe.releaseKey, last)
    await PendingLadder.record([{ probe, status: 'missing' }])
    const signature = await db
      .from('pending_signatures')
      .where({ tool: 'other', group_id: groupId, poster: '' })
      .firstOrFail()
    assert.include(signature, { probed: 100, named: 0, dropped: true })

    const waiting = await pending('00000000000000000000000000000010', [
      { kind: 'lone', messageId: '<w@x>', file: 'w' },
    ])
    const other = await pending(
      '00000000000000000000000000000011',
      [{ kind: 'lone', messageId: '<o@x>', file: 'o' }],
      { tool: 'nyuu' }
    )
    // samplePercent 0 in the test rules: no explored sample.
    await db.table('rule_sets').insert({
      version: 99,
      rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }),
      note: 'test',
    })
    assert.equal(await PendingLadder.discard(), 1)
    const statusOf = async (key: string) =>
      (
        await db
          .from('pending_releases')
          .where('release_key', Buffer.from(key, 'hex'))
          .firstOrFail()
      ).status
    assert.equal(await statusOf(waiting), PENDING.discarded)
    assert.equal(await statusOf(other), PENDING.waiting)
  })

  test('a probe given back goes to the queue again at the same step', async ({ assert }) => {
    const key = await pending('00000000000000000000000000000020', [
      { kind: 'lone', messageId: '<g@x>', file: 'g' },
    ])
    const [probe] = await PendingLadder.claim(1)
    await PendingLadder.record([{ probe: probe as PendingProbe, status: 'waiting' }])
    assert.include(
      await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(),
      { status: PENDING.waiting, tries: 0 }
    )
    assert.lengthOf(await db.from('pending_signatures'), 0)
  })
})