Voyager 3 schema: the slim catalog (migration 6000, the reset) and blocks (6100)
releases rebuilt without its text copies: poster_id, group_set_id, bytea keys, source smallint, edge, settled_at as the finish. posters by id, group_sets. blocks as the lease unit, ranges.block_id. fragments, range_blocks, release_final, the settler and the fragment merger gone; release_chunks, rule_counts and rule_samples keyed by block. Cold tables on CHUNK_TABLESPACE. Migration 6000 truncates the catalog and refuses without VOYAGER3_RESET=1. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
1a448f37f9
commit
346493b6c3
17 files changed
+527
-545
No files matched your search
@@ -3,12 +3,13 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import { ReleaseJudge } from '#services/release_judge'
|
||||
import SearchService from '#services/search_service'
|
||||
import RangeBlocks from '#services/range_blocks'
|
||||
|
||||
test.group('ReleaseJudge', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('judges dead, fake, repeated, locked and unusable releases, and takes a verdict back', async ({ assert }) => {
|
||||
test('judges dead, fake, repeated, locked and unusable releases, and takes a verdict back', async ({
|
||||
assert,
|
||||
}) => {
|
||||
let n = 0
|
||||
const settled = new Date(Date.now() - 3 * 86400_000)
|
||||
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
||||
@@ -16,19 +17,40 @@ test.group('ReleaseJudge', (group) => {
|
||||
const key = `jj${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', group_id: 1, files: 3, posted_at: settled, readable: true, settled_at: settled, ...extra })
|
||||
.insert({
|
||||
release_key: key,
|
||||
identity_key: key,
|
||||
name,
|
||||
source: 'stem',
|
||||
groups: '[]',
|
||||
group_id: 1,
|
||||
files: 3,
|
||||
posted_at: settled,
|
||||
readable: true,
|
||||
settled_at: settled,
|
||||
...extra,
|
||||
})
|
||||
.returning('id')
|
||||
return Number(row.id)
|
||||
}
|
||||
let candidate = 9000
|
||||
const probe = async (releaseId: number, status: string) => {
|
||||
candidate++
|
||||
await db.table('probe_candidates').insert({ id: candidate, release_id: releaseId, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status })
|
||||
await db.table('probe_candidates').insert({
|
||||
id: candidate,
|
||||
release_id: releaseId,
|
||||
kind: 'lone',
|
||||
message_id: `<${candidate}@x>`,
|
||||
file: 'x',
|
||||
status,
|
||||
})
|
||||
return candidate
|
||||
}
|
||||
const inspect = async (releaseId: number, kind: string, result: Record<string, unknown>) => {
|
||||
const id = await probe(releaseId, 'done')
|
||||
await db.table('release_inspections').insert({ release_id: releaseId, candidate_id: id, kind, result: JSON.stringify(result) })
|
||||
await db
|
||||
.table('release_inspections')
|
||||
.insert({ release_id: releaseId, candidate_id: id, kind, result: JSON.stringify(result) })
|
||||
}
|
||||
|
||||
const dead = await release('Gone.Movie.2020.1080p')
|
||||
@@ -39,7 +61,11 @@ test.group('ReleaseJudge', (group) => {
|
||||
await probe(halfDead, 'done')
|
||||
|
||||
const fakeArchive = await release('Das.A-Team.S02E23.Das.Spiel.ist.aus')
|
||||
await inspect(fakeArchive, 'archive', { type: 'rar4', names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'], entries: 1 })
|
||||
await inspect(fakeArchive, 'archive', {
|
||||
type: 'rar4',
|
||||
names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'],
|
||||
entries: 1,
|
||||
})
|
||||
const fakeFile = await release('Some.Film.2021.1080p.WEB-DL')
|
||||
// Only "H.264" and "WEB-DL" say it's media here (a "?" in the pattern never matched).
|
||||
const fakeCodec = await release('Some.Show.H.264.WEB-DL')
|
||||
@@ -54,34 +80,71 @@ test.group('ReleaseJudge', (group) => {
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const id = await release(`kantodo${i}`)
|
||||
repeatedIds.push(id)
|
||||
await db.table('file_hashes').insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 })
|
||||
await db
|
||||
.table('file_hashes')
|
||||
.insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 })
|
||||
}
|
||||
|
||||
const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', { poster: '@_CelebMovie_@', source: 'subject' })
|
||||
const celeborn = await release('Some.Album.2010', { poster: 'celeborn@not.here (Celeborn)', source: 'subject' })
|
||||
const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', {
|
||||
poster: '@_CelebMovie_@',
|
||||
source: 'subject',
|
||||
})
|
||||
const celeborn = await release('Some.Album.2010', {
|
||||
poster: 'celeborn@not.here (Celeborn)',
|
||||
source: 'subject',
|
||||
})
|
||||
// Scraped posters are found through posters (the batch store adds each release's).
|
||||
await db.table('posters').multiInsert([{ poster: '@_CelebMovie_@' }, { poster: 'celeborn@not.here (Celeborn)' }])
|
||||
await db
|
||||
.table('posters')
|
||||
.multiInsert([{ poster: '@_CelebMovie_@' }, { poster: 'celeborn@not.here (Celeborn)' }])
|
||||
const locked = await release('Locked.Thing', { passworded: true })
|
||||
|
||||
const unlocked = await release('Known.Password', { passworded: true })
|
||||
await db.table('release_passwords').insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })
|
||||
await db
|
||||
.table('release_passwords')
|
||||
.insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })
|
||||
|
||||
// A block of done ranges around the unusable ones (release_final); one beside an unread range isn't judged yet.
|
||||
await db.table('groups').insert({ id: 1, name: 'alt.binaries.judged', priority: 1 }).onConflict('id').ignore()
|
||||
await db
|
||||
.table('groups')
|
||||
.insert({ id: 1, name: 'alt.binaries.judged', priority: 1 })
|
||||
.onConflict('id')
|
||||
.ignore()
|
||||
await db.table('ranges').multiInsert([
|
||||
{ group_id: 1, start_id: 1, end_id: 1000, next_id: 1001, status: 'done', posted_low: new Date(settled.getTime() - 86400_000), posted_high: new Date(settled.getTime() + 86400_000) },
|
||||
{
|
||||
group_id: 1,
|
||||
start_id: 1,
|
||||
end_id: 1000,
|
||||
next_id: 1001,
|
||||
status: 'done',
|
||||
posted_low: new Date(settled.getTime() - 86400_000),
|
||||
posted_high: new Date(settled.getTime() + 86400_000),
|
||||
},
|
||||
{ group_id: 1, start_id: 1001, end_id: 2000, next_id: 1001, status: 'planned' },
|
||||
])
|
||||
await RangeBlocks.rebuild()
|
||||
const placed = { first_article: 100, last_article: 110 }
|
||||
const unusable = await release('ab12CD34ef56GH78ij90', { readable: false, ...placed })
|
||||
const pending = await release('pp12CD34ef56GH78ij90', { readable: false, first_article: 990, last_article: 1000, posted_at: new Date(settled.getTime() + 86400_000 - 3600_000) })
|
||||
const fresh = await release('zz12CD34ef56GH78ij90', { readable: false, settled_at: new Date(), ...placed })
|
||||
const pending = await release('pp12CD34ef56GH78ij90', {
|
||||
readable: false,
|
||||
first_article: 990,
|
||||
last_article: 1000,
|
||||
posted_at: new Date(settled.getTime() + 86400_000 - 3600_000),
|
||||
})
|
||||
const fresh = await release('zz12CD34ef56GH78ij90', {
|
||||
readable: false,
|
||||
settled_at: new Date(),
|
||||
...placed,
|
||||
})
|
||||
const spotted = await release('yy12CD34ef56GH78ij90', { readable: false, ...placed })
|
||||
await db.table('release_spots').insert({ release_id: spotted, spot_id: 's@x', files: 1 })
|
||||
|
||||
const round = await new ReleaseJudge().full()
|
||||
const verdicts = new Map((await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [Number(v.release_id), v.rule]))
|
||||
const verdicts = new Map(
|
||||
(await db.from('release_verdicts').select('release_id', 'rule')).map((v) => [
|
||||
Number(v.release_id),
|
||||
v.rule,
|
||||
])
|
||||
)
|
||||
assert.equal(verdicts.get(dead), 'dead')
|
||||
assert.isUndefined(verdicts.get(halfDead))
|
||||
assert.equal(verdicts.get(fakeArchive), 'fake')
|
||||
@@ -106,7 +169,9 @@ test.group('ReleaseJudge', (group) => {
|
||||
assert.equal((await SearchService.search({ q: 'Locked', judged: true })).total, 1)
|
||||
|
||||
// A password found: the next round takes the verdict back.
|
||||
await db.table('release_passwords').insert({ release_id: locked, password: 'found', source: 'nzb' })
|
||||
await db
|
||||
.table('release_passwords')
|
||||
.insert({ release_id: locked, password: 'found', source: 'nzb' })
|
||||
const again = await new ReleaseJudge().full()
|
||||
assert.equal(again.rules.locked.removed, 1)
|
||||
// Verdicts that stay are not written again.
|
||||
@@ -115,7 +180,9 @@ test.group('ReleaseJudge', (group) => {
|
||||
assert.isNull(await db.from('release_verdicts').where('release_id', locked).first())
|
||||
})
|
||||
|
||||
test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({ assert }) => {
|
||||
test('judges the releases whose facts changed, and those a hash of theirs tipped over', async ({
|
||||
assert,
|
||||
}) => {
|
||||
let n = 0
|
||||
const settled = new Date(Date.now() - 3 * 86400_000)
|
||||
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
||||
@@ -123,12 +190,24 @@ test.group('ReleaseJudge', (group) => {
|
||||
const key = `jc${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', files: 3, posted_at: settled, readable: true, settled_at: settled, ...extra })
|
||||
.insert({
|
||||
release_key: key,
|
||||
identity_key: key,
|
||||
name,
|
||||
source: 'stem',
|
||||
groups: '[]',
|
||||
files: 3,
|
||||
posted_at: settled,
|
||||
readable: true,
|
||||
settled_at: settled,
|
||||
...extra,
|
||||
})
|
||||
.returning('id')
|
||||
return Number(row.id)
|
||||
}
|
||||
const judge = new ReleaseJudge()
|
||||
const verdict = async (id: number) => (await db.from('release_verdicts').where('release_id', id).first())?.rule
|
||||
const verdict = async (id: number) =>
|
||||
(await db.from('release_verdicts').where('release_id', id).first())?.rule
|
||||
const dirty = async () => Number((await db.from('judge_dirty').count('* as n'))[0].n)
|
||||
|
||||
// New releases are noted; a passworded one is locked, until a password turns up.
|
||||
@@ -138,20 +217,31 @@ test.group('ReleaseJudge', (group) => {
|
||||
assert.equal(await verdict(locked), 'locked')
|
||||
assert.equal(round.rules.locked.added, 1)
|
||||
assert.equal(await dirty(), 0)
|
||||
await db.table('release_passwords').insert({ release_id: locked, password: 'pw', source: 'nzb' })
|
||||
await db
|
||||
.table('release_passwords')
|
||||
.insert({ release_id: locked, password: 'pw', source: 'nzb' })
|
||||
round = await judge.changed()
|
||||
assert.isUndefined(await verdict(locked))
|
||||
assert.equal(round.rules.locked.removed, 1)
|
||||
|
||||
// Probes: a release whose probed articles are all missing is dead once the prober's statuses come in.
|
||||
const dead = await release('Gone.Movie.2020.1080p')
|
||||
const ids = await db.table('probe_candidates').multiInsert([
|
||||
{ release_id: dead, kind: 'lone', message_id: '<jc1@x>', file: 'a', status: 'waiting' },
|
||||
{ release_id: dead, kind: 'lone', message_id: '<jc2@x>', file: 'b', status: 'waiting' },
|
||||
]).returning('id')
|
||||
const ids = await db
|
||||
.table('probe_candidates')
|
||||
.multiInsert([
|
||||
{ release_id: dead, kind: 'lone', message_id: '<jc1@x>', file: 'a', status: 'waiting' },
|
||||
{ release_id: dead, kind: 'lone', message_id: '<jc2@x>', file: 'b', status: 'waiting' },
|
||||
])
|
||||
.returning('id')
|
||||
await judge.changed()
|
||||
assert.isUndefined(await verdict(dead))
|
||||
await db.from('probe_candidates').whereIn('id', ids.map((r: { id: string }) => r.id)).update({ status: 'missing' })
|
||||
await db
|
||||
.from('probe_candidates')
|
||||
.whereIn(
|
||||
'id',
|
||||
ids.map((r: { id: string }) => r.id)
|
||||
)
|
||||
.update({ status: 'missing' })
|
||||
await judge.changed()
|
||||
assert.equal(await verdict(dead), 'dead')
|
||||
|
||||
@@ -159,13 +249,19 @@ test.group('ReleaseJudge', (group) => {
|
||||
// comes first), though only the 20th changed.
|
||||
const hash = 'b'.repeat(32)
|
||||
const holders: number[] = []
|
||||
for (let i = 0; i < 19; i++) holders.push(await release(`holder${i}`, i === 0 ? { passworded: true } : {}))
|
||||
for (const [i, id] of holders.entries()) await db.table('file_hashes').insert({ candidate_id: 30000 + i, release_id: id, hash16k: hash, size: 5000 })
|
||||
for (let i = 0; i < 19; i++)
|
||||
holders.push(await release(`holder${i}`, i === 0 ? { passworded: true } : {}))
|
||||
for (const [i, id] of holders.entries())
|
||||
await db
|
||||
.table('file_hashes')
|
||||
.insert({ candidate_id: 30000 + i, release_id: id, hash16k: hash, size: 5000 })
|
||||
await judge.changed()
|
||||
assert.equal(await verdict(holders[0]), 'locked')
|
||||
assert.isUndefined(await verdict(holders[1]))
|
||||
const twentieth = await release('holder19')
|
||||
await db.table('file_hashes').insert({ candidate_id: 30019, release_id: twentieth, hash16k: hash, size: 5000 })
|
||||
await db
|
||||
.table('file_hashes')
|
||||
.insert({ candidate_id: 30019, release_id: twentieth, hash16k: hash, size: 5000 })
|
||||
round = await judge.changed()
|
||||
assert.equal(round.releases, 20)
|
||||
const rules = new Set()
|
||||
@@ -173,7 +269,9 @@ test.group('ReleaseJudge', (group) => {
|
||||
assert.deepEqual([...rules], ['repeated'])
|
||||
// A 21st: the file was in 20 releases without it, so only it is judged.
|
||||
const twentyFirst = await release('holder20')
|
||||
await db.table('file_hashes').insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 })
|
||||
await db
|
||||
.table('file_hashes')
|
||||
.insert({ candidate_id: 30020, release_id: twentyFirst, hash16k: hash, size: 5000 })
|
||||
round = await judge.changed()
|
||||
assert.equal(round.releases, 1)
|
||||
assert.equal(await verdict(twentyFirst), 'repeated')
|
||||
|
||||
Reference in new issue
Block a user