Daan MeijerandClaude Fable 5.1
a71de56a7c
One file of one article, 2–100 kB, under a readable release-like name outside the music, e-book and comic groups, without an e-book, subtitle, crack or sidecar word: the 2015–17 flood's 1.5M rows that the edge indexed before its rule. The same tests as isTinyArchive in Postgres's regex dialect; no join on the candidates (98% of a sample have an archive candidate, the rest lone par2 files and scrambled names). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
322 lines
13 KiB
TypeScript
322 lines
13 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import { ReleaseJudge } from '#services/release_judge'
|
|
import SearchService from '#services/search_service'
|
|
import { insertRelease } from '#tests/functional/helpers/releases'
|
|
|
|
test.group('ReleaseJudge', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('judges dead, fake, repeated, locked and unusable releases, and takes a verdict back', async ({
|
|
assert,
|
|
}) => {
|
|
let n = 0
|
|
const settled = new Date(Date.now() - 3 * 86400_000)
|
|
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
|
n++
|
|
const key = `jj${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
|
const row = { id: await insertRelease({
|
|
release_key: key,
|
|
identity_key: key,
|
|
name,
|
|
source: 'stem',
|
|
groups: '[]',
|
|
group_id: 1,
|
|
files: 3,
|
|
posted_at: settled,
|
|
readable: true,
|
|
settled_at: settled,
|
|
...extra,
|
|
}) }
|
|
return Number(row.id)
|
|
}
|
|
let candidate = 9000
|
|
const probe = async (releaseId: number, status: string) => {
|
|
candidate++
|
|
await db.table('probe_candidates').insert({
|
|
id: candidate,
|
|
release_id: releaseId,
|
|
kind: 'lone',
|
|
message_id: `<${candidate}@x>`,
|
|
file: 'x',
|
|
status,
|
|
})
|
|
return candidate
|
|
}
|
|
const inspect = async (releaseId: number, kind: string, result: Record<string, unknown>) => {
|
|
const id = await probe(releaseId, 'done')
|
|
await db
|
|
.table('release_inspections')
|
|
.insert({ release_id: releaseId, candidate_id: id, kind, result: JSON.stringify(result) })
|
|
}
|
|
|
|
const dead = await release('Gone.Movie.2020.1080p')
|
|
await probe(dead, 'missing')
|
|
await probe(dead, 'missing')
|
|
const halfDead = await release('Half.Movie.2020.1080p')
|
|
await probe(halfDead, 'missing')
|
|
await probe(halfDead, 'done')
|
|
|
|
const fakeArchive = await release('Das.A-Team.S02E23.Das.Spiel.ist.aus')
|
|
await inspect(fakeArchive, 'archive', {
|
|
type: 'rar4',
|
|
names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'],
|
|
entries: 1,
|
|
})
|
|
const fakeFile = await release('Some.Film.2021.1080p.WEB-DL')
|
|
// Only "H.264" and "WEB-DL" say it's media here (a "?" in the pattern never matched).
|
|
const fakeCodec = await release('Some.Show.H.264.WEB-DL')
|
|
await inspect(fakeCodec, 'lone', { type: 'exe', yencName: 'setup.exe' })
|
|
await inspect(fakeFile, 'lone', { type: 'exe', yencName: 'Some.Film.2021.1080p.WEB-DL.mkv' })
|
|
// A program under a program's name is software, not fake.
|
|
const software = await release('LimeWire 4.19.6')
|
|
await inspect(software, 'archive', { type: 'rar4', names: ['LimeWire 4.19.6.exe'], entries: 1 })
|
|
|
|
const hash = 'a'.repeat(32)
|
|
const repeatedIds: number[] = []
|
|
for (let i = 0; i < 20; i++) {
|
|
const id = await release(`kantodo${i}`)
|
|
repeatedIds.push(id)
|
|
await db
|
|
.table('file_hashes')
|
|
.insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 })
|
|
}
|
|
|
|
const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', {
|
|
poster: '@_CelebMovie_@',
|
|
source: 'subject',
|
|
})
|
|
const celeborn = await release('Some.Album.2010', {
|
|
poster: 'celeborn@not.here (Celeborn)',
|
|
source: 'subject',
|
|
})
|
|
const locked = await release('Locked.Thing', { passworded: true })
|
|
|
|
const unlocked = await release('Known.Password', { passworded: true })
|
|
await db
|
|
.table('release_passwords')
|
|
.insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })
|
|
|
|
// Unusable once finished (the window closed on it); a provisional tail release isn't judged yet.
|
|
await db
|
|
.table('groups')
|
|
.insert({ id: 1, name: 'alt.binaries.judged', priority: 1 })
|
|
.onConflict('id')
|
|
.ignore()
|
|
const placed = { first_article: 100, last_article: 110 }
|
|
const unusable = await release('ab12CD34ef56GH78ij90', { readable: false, ...placed })
|
|
const pending = await release('pp12CD34ef56GH78ij90', {
|
|
readable: false,
|
|
first_article: 990,
|
|
last_article: 1000,
|
|
settled_at: null,
|
|
posted_at: new Date(settled.getTime() + 86400_000 - 3600_000),
|
|
})
|
|
const fresh = await release('zz12CD34ef56GH78ij90', {
|
|
readable: false,
|
|
settled_at: new Date(),
|
|
...placed,
|
|
})
|
|
const spotted = await release('yy12CD34ef56GH78ij90', { readable: false, ...placed })
|
|
await db.table('release_spots').insert({ release_id: spotted, spot_id: 's@x', files: 1 })
|
|
|
|
const round = await new ReleaseJudge().full()
|
|
const verdicts = new Map(
|
|
(await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [
|
|
Number(v.release_id),
|
|
v.rule,
|
|
])
|
|
)
|
|
assert.equal(verdicts.get(dead), 'dead')
|
|
assert.isUndefined(verdicts.get(halfDead))
|
|
assert.equal(verdicts.get(fakeArchive), 'fake')
|
|
assert.equal(verdicts.get(fakeFile), 'fake')
|
|
assert.equal(verdicts.get(fakeCodec), 'fake')
|
|
assert.isUndefined(verdicts.get(software))
|
|
assert.deepEqual([...new Set(repeatedIds.map((id) => verdicts.get(id)))], ['repeated'])
|
|
assert.equal(verdicts.get(scraped), 'scraped')
|
|
assert.isUndefined(verdicts.get(celeborn))
|
|
assert.equal(verdicts.get(locked), 'locked')
|
|
assert.isUndefined(verdicts.get(unlocked))
|
|
assert.equal(verdicts.get(unusable), 'unusable')
|
|
assert.isUndefined(verdicts.get(pending))
|
|
assert.isUndefined(verdicts.get(fresh))
|
|
assert.isUndefined(verdicts.get(spotted))
|
|
assert.equal(round.rules.repeated.added, 20)
|
|
assert.equal(round.rules.scraped.matched, 1)
|
|
|
|
// Search leaves them out, unless asked.
|
|
const found = await SearchService.search({ q: 'Locked' })
|
|
assert.equal(found.total, 0)
|
|
assert.equal((await SearchService.search({ q: 'Locked', judged: true })).total, 1)
|
|
|
|
// A password found: the next round takes the verdict back.
|
|
await db
|
|
.table('release_passwords')
|
|
.insert({ release_id: locked, password: 'found', source: 'nzb' })
|
|
const again = await new ReleaseJudge().full()
|
|
assert.equal(again.rules.locked.removed, 1)
|
|
// Verdicts that stay are not written again.
|
|
assert.equal(again.rules.repeated.added, 0)
|
|
assert.equal(again.rules.fake.added, 0)
|
|
assert.isNull(await db.from('release_verdicts').where('release_id', locked).first())
|
|
})
|
|
|
|
test('judges tiny single-article files under release-like names', async ({ assert }) => {
|
|
let n = 0
|
|
const settled = new Date(Date.now() - 3 * 86400_000)
|
|
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
|
n++
|
|
const key = `jt${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
|
return Number(
|
|
await insertRelease({
|
|
release_key: key,
|
|
identity_key: key,
|
|
name,
|
|
source: 'stem',
|
|
groups: ['alt.binaries.boneless'],
|
|
files: 1,
|
|
segments: 1,
|
|
bytes: 15_000,
|
|
posted_at: settled,
|
|
readable: true,
|
|
settled_at: settled,
|
|
...extra,
|
|
})
|
|
)
|
|
}
|
|
const dotted = await release('Velvet.S02E20.GERMAN.720p.HDTV.x264-ACED')
|
|
const spaced = await release('Pacific Rim (2013) BDRip DVDr-sailo1')
|
|
const subs = await release('Some.Film.2019.German.Subs-XYZ')
|
|
const ebook = await release('Some.Book.2019.epub-GRP')
|
|
const music = await release('Some.Album.2019.FLAC-GRP', { groups: ['alt.binaries.sounds.flac'] })
|
|
const big = await release('Big.Film.2019.1080p-GRP', { bytes: 500_000 })
|
|
const twoFiles = await release('Two.Films.2019.1080p-GRP', { files: 2, segments: 2 })
|
|
const plain = await release('Holiday photo')
|
|
const sidecar = await release('Side.Car.2019.1080p-GRP', { sidecar_only: true })
|
|
const unreadable = await release('Xk9v.2019.Qpz-ab', { readable: false })
|
|
|
|
const round = await new ReleaseJudge().full()
|
|
const verdicts = new Map(
|
|
(await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [
|
|
Number(v.release_id),
|
|
v.rule,
|
|
])
|
|
)
|
|
assert.equal(verdicts.get(dotted), 'tiny')
|
|
assert.equal(verdicts.get(spaced), 'tiny')
|
|
for (const id of [subs, ebook, music, big, twoFiles, plain, sidecar, unreadable]) {
|
|
assert.notEqual(verdicts.get(id), 'tiny')
|
|
}
|
|
assert.equal(round.rules.tiny.added, 2)
|
|
})
|
|
|
|
test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({
|
|
assert,
|
|
}) => {
|
|
let n = 0
|
|
const settled = new Date(Date.now() - 3 * 86400_000)
|
|
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
|
n++
|
|
const key = `jc${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
|
const row = { id: await insertRelease({
|
|
release_key: key,
|
|
identity_key: key,
|
|
name,
|
|
source: 'stem',
|
|
groups: '[]',
|
|
files: 3,
|
|
posted_at: settled,
|
|
readable: true,
|
|
settled_at: settled,
|
|
...extra,
|
|
}) }
|
|
return Number(row.id)
|
|
}
|
|
const judge = new ReleaseJudge()
|
|
const verdict = async (id: number) =>
|
|
(await db.from('release_verdicts').where('release_id', id).first())?.rule
|
|
const dirty = async () => Number((await db.from('judge_dirty').count('* as n'))[0].n)
|
|
|
|
// New releases are noted; a passworded one is locked, until a password turns up.
|
|
const locked = await release('Locked.Thing', { passworded: true })
|
|
assert.isAbove(await dirty(), 0)
|
|
let round = await judge.changed()
|
|
assert.equal(await verdict(locked), 'locked')
|
|
assert.equal(round.rules.locked.added, 1)
|
|
assert.equal(await dirty(), 0)
|
|
await db
|
|
.table('release_passwords')
|
|
.insert({ release_id: locked, password: 'pw', source: 'nzb' })
|
|
round = await judge.changed()
|
|
assert.isUndefined(await verdict(locked))
|
|
assert.equal(round.rules.locked.removed, 1)
|
|
|
|
// Probes: a release whose probed articles are all missing is dead once the prober's statuses come in.
|
|
const dead = await release('Gone.Movie.2020.1080p')
|
|
const ids = await db
|
|
.table('probe_candidates')
|
|
.multiInsert([
|
|
{ release_id: dead, kind: 'lone', message_id: '<jc1@x>', file: 'a', status: 'waiting' },
|
|
{ release_id: dead, kind: 'lone', message_id: '<jc2@x>', file: 'b', status: 'waiting' },
|
|
])
|
|
.returning('id')
|
|
await judge.changed()
|
|
assert.isUndefined(await verdict(dead))
|
|
await db
|
|
.from('probe_candidates')
|
|
.whereIn(
|
|
'id',
|
|
ids.map((r: { id: string }) => r.id)
|
|
)
|
|
.update({ status: 'missing' })
|
|
await judge.changed()
|
|
assert.equal(await verdict(dead), 'dead')
|
|
|
|
// One file in 19 releases, one of them locked: the 20th makes them all repeated, the locked one too (repeated
|
|
// comes first), though only the 20th changed.
|
|
const hash = 'b'.repeat(32)
|
|
const holders: number[] = []
|
|
for (let i = 0; i < 19; i++)
|
|
holders.push(await release(`holder${i}`, i === 0 ? { passworded: true } : {}))
|
|
for (const [i, id] of holders.entries())
|
|
await db
|
|
.table('file_hashes')
|
|
.insert({ candidate_id: 30000 + i, release_id: id, hash16k: hash, size: 5000 })
|
|
await judge.changed()
|
|
assert.equal(await verdict(holders[0]), 'locked')
|
|
assert.isUndefined(await verdict(holders[1]))
|
|
const twentieth = await release('holder19')
|
|
await db
|
|
.table('file_hashes')
|
|
.insert({ candidate_id: 30019, release_id: twentieth, hash16k: hash, size: 5000 })
|
|
round = await judge.changed()
|
|
assert.equal(round.releases, 20)
|
|
const rules = new Set()
|
|
for (const id of [...holders, twentieth]) rules.add(await verdict(id))
|
|
assert.deepEqual([...rules], ['repeated'])
|
|
// A 21st: the file was in 20 releases without it, so only it is judged.
|
|
const twentyFirst = await release('holder20')
|
|
await db
|
|
.table('file_hashes')
|
|
.insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 })
|
|
round = await judge.changed()
|
|
assert.equal(round.releases, 1)
|
|
assert.equal(await verdict(twentyFirst), 'repeated')
|
|
|
|
// A verdict nothing noted stays until the full round; it's due a day after the last one.
|
|
const stale = await release('Fine.Release')
|
|
await judge.changed()
|
|
await db.table('release_verdicts').insert({ release_id: stale, rule: 'fake', rule_version: 1 })
|
|
await judge.changed()
|
|
assert.equal(await verdict(stale), 'fake')
|
|
await db.from('judge_state').update({ full_at: new Date(Date.now() - 2 * 86400_000) })
|
|
assert.isTrue(await judge.fullDue())
|
|
round = await judge.full()
|
|
assert.isUndefined(await verdict(stale))
|
|
assert.equal(await verdict(dead), 'dead')
|
|
assert.isFalse(await judge.fullDue())
|
|
})
|
|
})
|