import { test } from '@japa/runner' import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import { ReleaseJudge } from '#services/release_judge' import SearchService from '#services/search_service' import { insertRelease } from '#tests/functional/helpers/releases' test.group('ReleaseJudge', (group) => { group.each.setup(() => testUtils.db().withGlobalTransaction()) test('judges dead, fake, repeated, locked and unusable releases, and takes a verdict back', async ({ assert, }) => { let n = 0 const settled = new Date(Date.now() - 3 * 86400_000) const release = async (name: string, extra: Record = {}) => { n++ const key = `jj${String(n).padStart(6, '0')}`.padEnd(40, '0') const row = { id: await insertRelease({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', group_id: 1, files: 3, posted_at: settled, readable: true, settled_at: settled, ...extra, }) } return Number(row.id) } let candidate = 9000 const probe = async (releaseId: number, status: string) => { candidate++ await db.table('probe_candidates').insert({ id: candidate, release_id: releaseId, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status, }) return candidate } const inspect = async (releaseId: number, kind: string, result: Record) => { const id = await probe(releaseId, 'done') await db .table('release_inspections') .insert({ release_id: releaseId, candidate_id: id, kind, result: JSON.stringify(result) }) } const dead = await release('Gone.Movie.2020.1080p') await probe(dead, 'missing') await probe(dead, 'missing') const halfDead = await release('Half.Movie.2020.1080p') await probe(halfDead, 'missing') await probe(halfDead, 'done') const fakeArchive = await release('Das.A-Team.S02E23.Das.Spiel.ist.aus') await inspect(fakeArchive, 'archive', { type: 'rar4', names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'], entries: 1, }) const fakeFile = await release('Some.Film.2021.1080p.WEB-DL') // Only "H.264" and "WEB-DL" say it's media here (a "?" in the pattern never matched). const fakeCodec = await release('Some.Show.H.264.WEB-DL') await inspect(fakeCodec, 'lone', { type: 'exe', yencName: 'setup.exe' }) await inspect(fakeFile, 'lone', { type: 'exe', yencName: 'Some.Film.2021.1080p.WEB-DL.mkv' }) // A program under a program's name is software, not fake. const software = await release('LimeWire 4.19.6') await inspect(software, 'archive', { type: 'rar4', names: ['LimeWire 4.19.6.exe'], entries: 1 }) const hash = 'a'.repeat(32) const repeatedIds: number[] = [] for (let i = 0; i < 20; i++) { const id = await release(`kantodo${i}`) repeatedIds.push(id) await db .table('file_hashes') .insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 }) } const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', { poster: '@_CelebMovie_@', source: 'subject', }) const celeborn = await release('Some.Album.2010', { poster: 'celeborn@not.here (Celeborn)', source: 'subject', }) const locked = await release('Locked.Thing', { passworded: true }) const unlocked = await release('Known.Password', { passworded: true }) await db .table('release_passwords') .insert({ release_id: unlocked, password: 'pw', source: 'spot-text' }) // Unusable once finished (the window closed on it); a provisional tail release isn't judged yet. await db .table('groups') .insert({ id: 1, name: 'alt.binaries.judged', priority: 1 }) .onConflict('id') .ignore() const placed = { first_article: 100, last_article: 110 } const unusable = await release('ab12CD34ef56GH78ij90', { readable: false, ...placed }) const pending = await release('pp12CD34ef56GH78ij90', { readable: false, first_article: 990, last_article: 1000, settled_at: null, posted_at: new Date(settled.getTime() + 86400_000 - 3600_000), }) const fresh = await release('zz12CD34ef56GH78ij90', { readable: false, settled_at: new Date(), ...placed, }) const spotted = await release('yy12CD34ef56GH78ij90', { readable: false, ...placed }) await db.table('release_spots').insert({ release_id: spotted, spot_id: 's@x', files: 1 }) const round = await new ReleaseJudge().full() const verdicts = new Map( (await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [ Number(v.release_id), v.rule, ]) ) assert.equal(verdicts.get(dead), 'dead') assert.isUndefined(verdicts.get(halfDead)) assert.equal(verdicts.get(fakeArchive), 'fake') assert.equal(verdicts.get(fakeFile), 'fake') assert.equal(verdicts.get(fakeCodec), 'fake') assert.isUndefined(verdicts.get(software)) assert.deepEqual([...new Set(repeatedIds.map((id) => verdicts.get(id)))], ['repeated']) assert.equal(verdicts.get(scraped), 'scraped') assert.isUndefined(verdicts.get(celeborn)) assert.equal(verdicts.get(locked), 'locked') assert.isUndefined(verdicts.get(unlocked)) assert.equal(verdicts.get(unusable), 'unusable') assert.isUndefined(verdicts.get(pending)) assert.isUndefined(verdicts.get(fresh)) assert.isUndefined(verdicts.get(spotted)) assert.equal(round.rules.repeated.added, 20) assert.equal(round.rules.scraped.matched, 1) // Search leaves them out, unless asked. const found = await SearchService.search({ q: 'Locked' }) assert.equal(found.total, 0) assert.equal((await SearchService.search({ q: 'Locked', judged: true })).total, 1) // A password found: the next round takes the verdict back. await db .table('release_passwords') .insert({ release_id: locked, password: 'found', source: 'nzb' }) const again = await new ReleaseJudge().full() assert.equal(again.rules.locked.removed, 1) // Verdicts that stay are not written again. assert.equal(again.rules.repeated.added, 0) assert.equal(again.rules.fake.added, 0) assert.isNull(await db.from('release_verdicts').where('release_id', locked).first()) }) test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({ assert, }) => { let n = 0 const settled = new Date(Date.now() - 3 * 86400_000) const release = async (name: string, extra: Record = {}) => { n++ const key = `jc${String(n).padStart(6, '0')}`.padEnd(40, '0') const row = { id: await insertRelease({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', files: 3, posted_at: settled, readable: true, settled_at: settled, ...extra, }) } return Number(row.id) } const judge = new ReleaseJudge() const verdict = async (id: number) => (await db.from('release_verdicts').where('release_id', id).first())?.rule const dirty = async () => Number((await db.from('judge_dirty').count('* as n'))[0].n) // New releases are noted; a passworded one is locked, until a password turns up. const locked = await release('Locked.Thing', { passworded: true }) assert.isAbove(await dirty(), 0) let round = await judge.changed() assert.equal(await verdict(locked), 'locked') assert.equal(round.rules.locked.added, 1) assert.equal(await dirty(), 0) await db .table('release_passwords') .insert({ release_id: locked, password: 'pw', source: 'nzb' }) round = await judge.changed() assert.isUndefined(await verdict(locked)) assert.equal(round.rules.locked.removed, 1) // Probes: a release whose probed articles are all missing is dead once the prober's statuses come in. const dead = await release('Gone.Movie.2020.1080p') const ids = await db .table('probe_candidates') .multiInsert([ { release_id: dead, kind: 'lone', message_id: '', file: 'a', status: 'waiting' }, { release_id: dead, kind: 'lone', message_id: '', file: 'b', status: 'waiting' }, ]) .returning('id') await judge.changed() assert.isUndefined(await verdict(dead)) await db .from('probe_candidates') .whereIn( 'id', ids.map((r: { id: string }) => r.id) ) .update({ status: 'missing' }) await judge.changed() assert.equal(await verdict(dead), 'dead') // One file in 19 releases, one of them locked: the 20th makes them all repeated, the locked one too (repeated // comes first), though only the 20th changed. const hash = 'b'.repeat(32) const holders: number[] = [] for (let i = 0; i < 19; i++) holders.push(await release(`holder${i}`, i === 0 ? { passworded: true } : {})) for (const [i, id] of holders.entries()) await db .table('file_hashes') .insert({ candidate_id: 30000 + i, release_id: id, hash16k: hash, size: 5000 }) await judge.changed() assert.equal(await verdict(holders[0]), 'locked') assert.isUndefined(await verdict(holders[1])) const twentieth = await release('holder19') await db .table('file_hashes') .insert({ candidate_id: 30019, release_id: twentieth, hash16k: hash, size: 5000 }) round = await judge.changed() assert.equal(round.releases, 20) const rules = new Set() for (const id of [...holders, twentieth]) rules.add(await verdict(id)) assert.deepEqual([...rules], ['repeated']) // A 21st: the file was in 20 releases without it, so only it is judged. const twentyFirst = await release('holder20') await db .table('file_hashes') .insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 }) round = await judge.changed() assert.equal(round.releases, 1) assert.equal(await verdict(twentyFirst), 'repeated') // A verdict nothing noted stays until the full round; it's due a day after the last one. const stale = await release('Fine.Release') await judge.changed() await db.table('release_verdicts').insert({ release_id: stale, rule: 'fake', rule_version: 1 }) await judge.changed() assert.equal(await verdict(stale), 'fake') await db.from('judge_state').update({ full_at: new Date(Date.now() - 2 * 86400_000) }) assert.isTrue(await judge.fullDue()) round = await judge.full() assert.isUndefined(await verdict(stale)) assert.equal(await verdict(dead), 'dead') assert.isFalse(await judge.fullDue()) }) })