usenet-indexer/tests/functional/release_judge.spec.ts

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import { ReleaseJudge } from '#services/release_judge'
import SearchService from '#services/search_service'
import { insertRelease } from '#tests/functional/helpers/releases'

test.group('ReleaseJudge', (group) => {
  group.each.setup(() => testUtils.db().withGlobalTransaction())

  test('judges dead, fake, repeated, locked and unusable releases, and takes a verdict back', async ({
    assert,
  }) => {
    let n = 0
    const settled = new Date(Date.now() - 3 * 86400_000)
    const release = async (name: string, extra: Record<string, unknown> = {}) => {
      n++
      const key = `jj${String(n).padStart(6, '0')}`.padEnd(40, '0')
      const row = { id: await insertRelease({
          release_key: key,
          identity_key: key,
          name,
          source: 'stem',
          groups: '[]',
          group_id: 1,
          files: 3,
          posted_at: settled,
          readable: true,
          settled_at: settled,
          ...extra,
        }) }
      return Number(row.id)
    }
    let candidate = 9000
    const probe = async (releaseId: number, status: string) => {
      candidate++
      await db.table('probe_candidates').insert({
        id: candidate,
        release_id: releaseId,
        kind: 'lone',
        message_id: `<${candidate}@x>`,
        file: 'x',
        status,
      })
      return candidate
    }
    const inspect = async (releaseId: number, kind: string, result: Record<string, unknown>) => {
      const id = await probe(releaseId, 'done')
      await db
        .table('release_inspections')
        .insert({ release_id: releaseId, candidate_id: id, kind, result: JSON.stringify(result) })
    }

    const dead = await release('Gone.Movie.2020.1080p')
    await probe(dead, 'missing')
    await probe(dead, 'missing')
    const halfDead = await release('Half.Movie.2020.1080p')
    await probe(halfDead, 'missing')
    await probe(halfDead, 'done')

    const fakeArchive = await release('Das.A-Team.S02E23.Das.Spiel.ist.aus')
    await inspect(fakeArchive, 'archive', {
      type: 'rar4',
      names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'],
      entries: 1,
    })
    const fakeFile = await release('Some.Film.2021.1080p.WEB-DL')
    // Only "H.264" and "WEB-DL" say it's media here (a "?" in the pattern never matched).
    const fakeCodec = await release('Some.Show.H.264.WEB-DL')
    await inspect(fakeCodec, 'lone', { type: 'exe', yencName: 'setup.exe' })
    await inspect(fakeFile, 'lone', { type: 'exe', yencName: 'Some.Film.2021.1080p.WEB-DL.mkv' })
    // A program under a program's name is software, not fake.
    const software = await release('LimeWire 4.19.6')
    await inspect(software, 'archive', { type: 'rar4', names: ['LimeWire 4.19.6.exe'], entries: 1 })

    const hash = 'a'.repeat(32)
    const repeatedIds: number[] = []
    for (let i = 0; i < 20; i++) {
      const id = await release(`kantodo${i}`)
      repeatedIds.push(id)
      await db
        .table('file_hashes')
        .insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 })
    }

    const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', {
      poster: '@_CelebMovie_@',
      source: 'subject',
    })
    const celeborn = await release('Some.Album.2010', {
      poster: 'celeborn@not.here (Celeborn)',
      source: 'subject',
    })
    const locked = await release('Locked.Thing', { passworded: true })

    const unlocked = await release('Known.Password', { passworded: true })
    await db
      .table('release_passwords')
      .insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })

    // Unusable once finished (the window closed on it); a provisional tail release isn't judged yet.
    await db
      .table('groups')
      .insert({ id: 1, name: 'alt.binaries.judged', priority: 1 })
      .onConflict('id')
      .ignore()
    const placed = { first_article: 100, last_article: 110 }
    const unusable = await release('ab12CD34ef56GH78ij90', { readable: false, ...placed })
    const pending = await release('pp12CD34ef56GH78ij90', {
      readable: false,
      first_article: 990,
      last_article: 1000,
      settled_at: null,
      posted_at: new Date(settled.getTime() + 86400_000 - 3600_000),
    })
    const fresh = await release('zz12CD34ef56GH78ij90', {
      readable: false,
      settled_at: new Date(),
      ...placed,
    })
    const spotted = await release('yy12CD34ef56GH78ij90', { readable: false, ...placed })
    await db.table('release_spots').insert({ release_id: spotted, spot_id: 's@x', files: 1 })

    const round = await new ReleaseJudge().full()
    const verdicts = new Map(
      (await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [
        Number(v.release_id),
        v.rule,
      ])
    )
    assert.equal(verdicts.get(dead), 'dead')
    assert.isUndefined(verdicts.get(halfDead))
    assert.equal(verdicts.get(fakeArchive), 'fake')
    assert.equal(verdicts.get(fakeFile), 'fake')
    assert.equal(verdicts.get(fakeCodec), 'fake')
    assert.isUndefined(verdicts.get(software))
    assert.deepEqual([...new Set(repeatedIds.map((id) => verdicts.get(id)))], ['repeated'])
    assert.equal(verdicts.get(scraped), 'scraped')
    assert.isUndefined(verdicts.get(celeborn))
    assert.equal(verdicts.get(locked), 'locked')
    assert.isUndefined(verdicts.get(unlocked))
    assert.equal(verdicts.get(unusable), 'unusable')
    assert.isUndefined(verdicts.get(pending))
    assert.isUndefined(verdicts.get(fresh))
    assert.isUndefined(verdicts.get(spotted))
    assert.equal(round.rules.repeated.added, 20)
    assert.equal(round.rules.scraped.matched, 1)

    // Search leaves them out, unless asked.
    const found = await SearchService.search({ q: 'Locked' })
    assert.equal(found.total, 0)
    assert.equal((await SearchService.search({ q: 'Locked', judged: true })).total, 1)

    // A password found: the next round takes the verdict back.
    await db
      .table('release_passwords')
      .insert({ release_id: locked, password: 'found', source: 'nzb' })
    const again = await new ReleaseJudge().full()
    assert.equal(again.rules.locked.removed, 1)
    // Verdicts that stay are not written again.
    assert.equal(again.rules.repeated.added, 0)
    assert.equal(again.rules.fake.added, 0)
    assert.isNull(await db.from('release_verdicts').where('release_id', locked).first())
  })

  test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({
    assert,
  }) => {
    let n = 0
    const settled = new Date(Date.now() - 3 * 86400_000)
    const release = async (name: string, extra: Record<string, unknown> = {}) => {
      n++
      const key = `jc${String(n).padStart(6, '0')}`.padEnd(40, '0')
      const row = { id: await insertRelease({
          release_key: key,
          identity_key: key,
          name,
          source: 'stem',
          groups: '[]',
          files: 3,
          posted_at: settled,
          readable: true,
          settled_at: settled,
          ...extra,
        }) }
      return Number(row.id)
    }
    const judge = new ReleaseJudge()
    const verdict = async (id: number) =>
      (await db.from('release_verdicts').where('release_id', id).first())?.rule
    const dirty = async () => Number((await db.from('judge_dirty').count('* as n'))[0].n)

    // New releases are noted; a passworded one is locked, until a password turns up.
    const locked = await release('Locked.Thing', { passworded: true })
    assert.isAbove(await dirty(), 0)
    let round = await judge.changed()
    assert.equal(await verdict(locked), 'locked')
    assert.equal(round.rules.locked.added, 1)
    assert.equal(await dirty(), 0)
    await db
      .table('release_passwords')
      .insert({ release_id: locked, password: 'pw', source: 'nzb' })
    round = await judge.changed()
    assert.isUndefined(await verdict(locked))
    assert.equal(round.rules.locked.removed, 1)

    // Probes: a release whose probed articles are all missing is dead once the prober's statuses come in.
    const dead = await release('Gone.Movie.2020.1080p')
    const ids = await db
      .table('probe_candidates')
      .multiInsert([
        { release_id: dead, kind: 'lone', message_id: '<jc1@x>', file: 'a', status: 'waiting' },
        { release_id: dead, kind: 'lone', message_id: '<jc2@x>', file: 'b', status: 'waiting' },
      ])
      .returning('id')
    await judge.changed()
    assert.isUndefined(await verdict(dead))
    await db
      .from('probe_candidates')
      .whereIn(
        'id',
        ids.map((r: { id: string }) => r.id)
      )
      .update({ status: 'missing' })
    await judge.changed()
    assert.equal(await verdict(dead), 'dead')

    // One file in 19 releases, one of them locked: the 20th makes them all repeated, the locked one too (repeated
    // comes first), though only the 20th changed.
    const hash = 'b'.repeat(32)
    const holders: number[] = []
    for (let i = 0; i < 19; i++)
      holders.push(await release(`holder${i}`, i === 0 ? { passworded: true } : {}))
    for (const [i, id] of holders.entries())
      await db
        .table('file_hashes')
        .insert({ candidate_id: 30000 + i, release_id: id, hash16k: hash, size: 5000 })
    await judge.changed()
    assert.equal(await verdict(holders[0]), 'locked')
    assert.isUndefined(await verdict(holders[1]))
    const twentieth = await release('holder19')
    await db
      .table('file_hashes')
      .insert({ candidate_id: 30019, release_id: twentieth, hash16k: hash, size: 5000 })
    round = await judge.changed()
    assert.equal(round.releases, 20)
    const rules = new Set()
    for (const id of [...holders, twentieth]) rules.add(await verdict(id))
    assert.deepEqual([...rules], ['repeated'])
    // A 21st: the file was in 20 releases without it, so only it is judged.
    const twentyFirst = await release('holder20')
    await db
      .table('file_hashes')
      .insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 })
    round = await judge.changed()
    assert.equal(round.releases, 1)
    assert.equal(await verdict(twentyFirst), 'repeated')

    // A verdict nothing noted stays until the full round; it's due a day after the last one.
    const stale = await release('Fine.Release')
    await judge.changed()
    await db.table('release_verdicts').insert({ release_id: stale, rule: 'fake', rule_version: 1 })
    await judge.changed()
    assert.equal(await verdict(stale), 'fake')
    await db.from('judge_state').update({ full_at: new Date(Date.now() - 2 * 86400_000) })
    assert.isTrue(await judge.fullDue())
    round = await judge.full()
    assert.isUndefined(await verdict(stale))
    assert.equal(await verdict(dead), 'dead')
    assert.isFalse(await judge.fullDue())
  })
})