usenet-indexer/tests/functional/pending_ladder.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import PendingLadder, { PENDING, byEffort, type PendingProbe } from '#services/pending_ladder'
import { identityKey, releaseKey } from '#lib/release/keys'

let groupId = 0
let n = 0

async function pending(
  name: string,
  candidates: { kind: string; messageId: string; file: string; bytes?: number }[],
  extra: Record<string, unknown> = {}
) {
  n++
  const key = releaseKey(name, groupId, n)
  await db.table('pending_releases').insert({
    release_key: key,
    identity_key: identityKey(name),
    group_id: groupId,
    name,
    poster: 'Bob <bob@home.me>',
    first_article: n,
    last_article: n,
    files: 1,
    segments: 10,
    bytes: 1000,
    tool: 'other',
    candidates: JSON.stringify(candidates),
    ...extra,
  })
  return key.toString('hex')
}

const inspection = (yencName: string, names?: string[]) => ({
  yencName,
  size: 1000,
  type: null,
  names,
  entries: names?.length,
})

test.group('Pending ladder', (group) => {
  group.each.setup(async () => {
    const rollback = await testUtils.db().withGlobalTransaction()
    groupId = Number(
      (
        await db
          .table('groups')
          .insert({ name: 'alt.binaries.pending', priority: 1 })
          .returning('id')
      )[0].id
    )
    n = 0
    return rollback
  })

  test('targets are climbed by effort: a par2 before an archive before a lone file', ({
    assert,
  }) => {
    assert.deepEqual(
      byEffort([
        { kind: 'lone', messageId: 'l', file: 'l' },
        { kind: '7z-tail', messageId: 't', file: 't' },
        { kind: 'archive', messageId: 'a', file: 'a' },
        { kind: 'par2', messageId: 'p', file: 'p' },
      ]).map((t) => t.kind),
      ['par2', 'archive', 'lone', '7z-tail']
    )
  })

  test('a claim takes the oldest waiting releases, their cheapest target first; outcomes move them up or to the end', async ({
    assert,
  }) => {
    const twoTargets = await pending('a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4', [
      { kind: 'archive', messageId: '<a@x>', file: 'x.rar' },
      { kind: 'par2', messageId: '<p@x>', file: 'x.par2' },
    ])
    const lone = await pending('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3', [
      { kind: 'lone', messageId: '<l@x>', file: 'f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3' },
    ])
    const none = await pending('00000000000000000000000000000001', [])
    const provisional = await pending(
      '00000000000000000000000000000002',
      [{ kind: 'lone', messageId: '<v@x>', file: 'v' }],
      { finished: false }
    )

    const probes = await PendingLadder.claim(10)
    assert.deepEqual(
      probes.map((p) => [p.releaseKey, p.kind, p.tries, p.targets]),
      [
        [twoTargets, 'par2', 0, 2],
        [lone, 'lone', 0, 1],
      ]
    )
    const statusOf = async (key: string) =>
      (await db
        .from('pending_releases')
        .where('release_key', Buffer.from(key, 'hex'))
        .firstOrFail()) as any
    assert.equal((await statusOf(twoTargets)).status, PENDING.taken)
    assert.equal((await statusOf(none)).status, PENDING.waiting)
    assert.equal((await statusOf(provisional)).status, PENDING.waiting)

    // The par2 names only random volumes: the next target. The lone file's yEnc name is its random name: exhausted.
    await PendingLadder.record([
      {
        probe: probes[0],
        status: 'done',
        inspection: inspection('x.par2', ['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4.part01.rar']),
      },
      {
        probe: probes[1],
        status: 'done',
        inspection: inspection('f6e5d4c3b2a1f6e5d4c3b2a1f6e5d4c3'),
      },
    ])
    const first = await statusOf(twoTargets)
    assert.include(first, { status: PENDING.waiting, tries: 1, named: null })
    assert.lengthOf(first.inspections, 1)
    assert.include(await statusOf(lone), { status: PENDING.exhausted, tries: 1 })

    // The archive's listing names it: for the namer.
    const [second] = await PendingLadder.claim(10)
    assert.deepEqual([second.releaseKey, second.kind, second.tries], [twoTargets, 'archive', 1])
    await PendingLadder.record([
      {
        probe: second,
        status: 'done',
        inspection: inspection('x.rar', ['Some.Movie.2016.1080p.mkv']),
      },
    ])
    assert.include(await statusOf(twoTargets), {
      status: PENDING.naming,
      named: 'Some.Movie.2016.1080p.mkv',
      tries: 0,
    })

    // The signature counted both ways: the poster's own and the tool's in the group.
    const signatures = await db.from('pending_signatures').orderBy('poster')
    assert.deepEqual(
      signatures.map((s) => [s.tool, s.poster, s.probed, s.named, s.dropped]),
      [
        ['other', '', 2, 1, false],
        ['other', 'Bob <bob@home.me>', 2, 1, false],
      ]
    )
  })

  test('a signature that names nothing is dropped, and its waiting releases discarded without a probe', async ({
    assert,
  }) => {
    // 100 climbs of the tool in the group, none named.
    await db
      .table('pending_signatures')
      .insert({ tool: 'other', group_id: groupId, poster: '', probed: 99, named: 0 })
    const last = await pending('00000000000000000000000000000009', [
      { kind: 'lone', messageId: '<z@x>', file: 'z' },
    ])
    const [probe] = await PendingLadder.claim(1)
    assert.equal(probe.releaseKey, last)
    await PendingLadder.record([{ probe, status: 'missing' }])
    const signature = await db
      .from('pending_signatures')
      .where({ tool: 'other', group_id: groupId, poster: '' })
      .firstOrFail()
    assert.include(signature, { probed: 100, named: 0, dropped: true })

    const waiting = await pending('00000000000000000000000000000010', [
      { kind: 'lone', messageId: '<w@x>', file: 'w' },
    ])
    const other = await pending(
      '00000000000000000000000000000011',
      [{ kind: 'lone', messageId: '<o@x>', file: 'o' }],
      { tool: 'nyuu' }
    )
    // samplePercent 0 in the test rules: no explored sample.
    await db.table('rule_sets').insert({
      version: 99,
      rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }),
      note: 'test',
    })
    assert.equal(await PendingLadder.discard(), 1)
    const statusOf = async (key: string) =>
      (
        await db
          .from('pending_releases')
          .where('release_key', Buffer.from(key, 'hex'))
          .firstOrFail()
      ).status
    assert.equal(await statusOf(waiting), PENDING.discarded)
    assert.equal(await statusOf(other), PENDING.waiting)
  })

  test('a probe given back goes to the queue again at the same step', async ({ assert }) => {
    const key = await pending('00000000000000000000000000000020', [
      { kind: 'lone', messageId: '<g@x>', file: 'g' },
    ])
    const [probe] = await PendingLadder.claim(1)
    await PendingLadder.record([{ probe: probe as PendingProbe, status: 'waiting' }])
    assert.include(
      await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(),
      { status: PENDING.waiting, tries: 0 }
    )
    assert.lengthOf(await db.from('pending_signatures'), 0)
  })
})

test.group('Pending ladder / classes', (group) => {
  group.each.setup(async () => {
    PendingLadder.forget()
    const rollback = await testUtils.db().withGlobalTransaction()
    groupId = Number(
      (
        await db
          .table('groups')
          .insert({ name: 'alt.binaries.classes', priority: 1 })
          .returning('id')
      )[0].id
    )
    n = 0
    return rollback
  })

  test('a claim takes the class with the best record first, the cheapest target within it, the rest oldest first', async ({
    assert,
  }) => {
    // Class 15 (1-10 MB hex, whole) named 9 of 10; class 16 (partial) named 0 of 50; class 0: rows from before.
    await db.table('pending_classes').multiInsert([
      { tool: 'other', group_id: groupId, klass: 15, probed: 10, named: 9, waiting: 2 },
      { tool: 'other', group_id: groupId, klass: 16, probed: 50, named: 0, waiting: 1 },
    ])
    const old = await pending('00000000000000000000000000000031', [
      { kind: 'lone', messageId: '<o@x>', file: 'o' },
    ])
    const poor = await pending(
      '00000000000000000000000000000032',
      [{ kind: 'lone', messageId: '<p@x>', file: 'p' }],
      { klass: 16 }
    )
    const bigTarget = await pending(
      '00000000000000000000000000000033',
      [{ kind: 'archive', messageId: '<b@x>', file: 'b.rar', bytes: 700_000 }],
      { klass: 15, target_bytes: 700_000 }
    )
    const smallTarget = await pending(
      '00000000000000000000000000000034',
      [{ kind: 'archive', messageId: '<s@x>', file: 's.rar', bytes: 50_000 }],
      { klass: 15, target_bytes: 50_000 }
    )
    const probes = await PendingLadder.claim(10)
    assert.deepEqual(
      probes.map((p) => [p.releaseKey, p.klass]),
      [
        [smallTarget, 15],
        [bigTarget, 15],
        [poor, 16],
        [old, 0],
      ]
    )
    const waiting = await db
      .from('pending_classes')
      .where({ group_id: groupId })
      .orderBy('klass')
      .select('klass', 'waiting')
    assert.deepEqual(
      waiting.map((c) => [c.klass, c.waiting]),
      [
        [15, 0],
        [16, 0],
      ]
    )
    // Outcomes count on the class; a probe given back waits again.
    await PendingLadder.record([
      {
        probe: probes[0],
        status: 'done',
        inspection: inspection('s.rar', ['Some.Movie.2016.1080p.mkv']),
      },
      { probe: probes[1], status: 'waiting' },
      { probe: probes[2], status: 'missing' },
    ])
    const classes = await db.from('pending_classes').where({ group_id: groupId }).orderBy('klass')
    assert.deepEqual(
      classes.map((c) => [c.klass, c.probed, c.named, c.waiting, c.dropped]),
      [
        [15, 11, 10, 1, false],
        [16, 51, 0, 0, false],
      ]
    )
  })

  test('a class that names nothing is dropped and its waiting releases discarded, the other classes untouched', async ({
    assert,
  }) => {
    await db
      .table('pending_classes')
      .insert({ tool: 'other', group_id: groupId, klass: 16, probed: 99, named: 0, waiting: 1 })
    const last = await pending(
      '00000000000000000000000000000041',
      [{ kind: 'lone', messageId: '<z@x>', file: 'z' }],
      { klass: 16 }
    )
    const [probe] = await PendingLadder.claim(1)
    assert.equal(probe.releaseKey, last)
    await PendingLadder.record([{ probe, status: 'missing' }])
    assert.include(
      await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail(),
      { probed: 100, named: 0, dropped: true }
    )
    const waiting = await pending(
      '00000000000000000000000000000042',
      [{ kind: 'lone', messageId: '<w@x>', file: 'w' }],
      { klass: 16 }
    )
    const other = await pending(
      '00000000000000000000000000000043',
      [{ kind: 'lone', messageId: '<o@x>', file: 'o' }],
      { klass: 15 }
    )
    await db.rawQuery(
      `UPDATE pending_classes SET waiting = waiting + 1 WHERE group_id = ? AND klass = 16`,
      [groupId]
    )
    await db.table('rule_sets').insert({
      version: 98,
      rules: JSON.stringify({ samplePercent: 0, droppedSignatures: [] }),
      note: 'test',
    })
    assert.equal(await PendingLadder.discard(), 1)
    const statusOf = async (key: string) =>
      (
        await db
          .from('pending_releases')
          .where('release_key', Buffer.from(key, 'hex'))
          .firstOrFail()
      ).status
    assert.equal(await statusOf(waiting), PENDING.discarded)
    assert.equal(await statusOf(other), PENDING.waiting)
    assert.equal(
      (await db.from('pending_classes').where({ group_id: groupId, klass: 16 }).firstOrFail())
        .waiting,
      0
    )
  })

  test('a twin by content names a release whose own probe did not', async ({ assert }) => {
    // A catalog release with a readable name holds the same first 16 KiB and size.
    const [poster] = await db.table('posters').insert({ poster: 'Twin <twin@x>' }).returning('id')
    const [set] = await db
      .table('group_sets')
      .insert({ groups: JSON.stringify(['alt.binaries.classes']) })
      .returning('id')
    const [twin] = await db
      .table('releases')
      .insert({
        release_key: Buffer.from('aa'.repeat(20), 'hex'),
        identity_key: Buffer.from('bb'.repeat(20), 'hex'),
        group_id: groupId,
        group_set_id: set.id,
        poster_id: poster.id,
        name: 'Some.Movie.2016.1080p.BluRay-GRP',
        source: 3,
        posted_at: new Date('2026-10-01T00:00:00Z'),
        first_article: 1,
        last_article: 2,
        files: 2,
        segments: 2,
        bytes: 1000,
        readable: true,
      })
      .returning('id')
    await db.table('file_hashes').insert({
      candidate_id: 777_001,
      release_id: twin.id,
      hash16k: 'deadbeef'.repeat(4),
      size: 1000,
    })
    const key = await pending('00000000000000000000000000000051', [
      { kind: 'lone', messageId: '<t@x>', file: '00000000000000000000000000000051' },
    ])
    const [probe] = await PendingLadder.claim(1)
    assert.equal(probe.releaseKey, key)
    await PendingLadder.record([
      {
        probe,
        status: 'done',
        inspection: {
          ...inspection('00000000000000000000000000000051'),
          hash16k: 'deadbeef'.repeat(4),
        },
      },
    ])
    assert.include(
      await db.from('pending_releases').where('release_key', Buffer.from(key, 'hex')).firstOrFail(),
      {
        status: PENDING.naming,
        named: 'Some.Movie.2016.1080p.BluRay-GRP',
      }
    )
  })
})