Daan MeijerandClaude Fable 5.1
2580f20a43
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
123 lines
7.4 KiB
TypeScript
123 lines
7.4 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import { Par2Pruner } from '#services/par2_pruner'
|
|
import { insertRelease } from '#tests/functional/helpers/releases'
|
|
|
|
test.group('Par2Pruner', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('drops the file lists of complete and judged sets, keeps the rest', async ({ assert }) => {
|
|
const old = new Date(Date.now() - 3 * 86400_000)
|
|
let n = 0
|
|
const release = async () => {
|
|
n++
|
|
const key = `pp${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
|
const row = { id: await insertRelease({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 2, readable: false }) }
|
|
return Number(row.id)
|
|
}
|
|
let candidate = 5000
|
|
const set = async (letter: string, holder: number, files: number, hashedIn: number | null, created = old) => {
|
|
const setId = letter.repeat(32)
|
|
for (let i = 0; i < files; i++) {
|
|
const h = `${letter}${i}`.padEnd(32, '0')
|
|
await db.table('par2_files').insert({ set_id: setId, file_id: h, hash16k: h, length: 1000 + i, name: `f${i}.rar` })
|
|
if (hashedIn && i < files) {
|
|
candidate++
|
|
await db.table('probe_candidates').insert({ id: candidate, release_id: hashedIn, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
|
|
await db.table('file_hashes').insert({ candidate_id: candidate, release_id: hashedIn, hash16k: h, size: 1000 + i })
|
|
}
|
|
}
|
|
await db.table('par2_sets').insert({ set_id: setId, release_id: holder, files, created_at: created })
|
|
return setId
|
|
}
|
|
const count = async (setId: string) => Number((await db.from('par2_files').where('set_id', setId).count('* as n'))[0].n)
|
|
|
|
// Every file's hash in the holder: complete.
|
|
const a = await release()
|
|
const complete = await set('a', a, 2, a)
|
|
// The files hashed in another release (not merged): kept.
|
|
const b = await release()
|
|
const apart = await set('b', b, 2, await release())
|
|
// Complete but recent: kept (its merges may come).
|
|
const c = await release()
|
|
const recent = await set('c', c, 2, c, new Date())
|
|
// Nothing hashed, its only holder dead: judged.
|
|
const d = await release()
|
|
const dead = await set('d', d, 2, null, new Date())
|
|
await db.table('release_verdicts').insert({ release_id: d, rule: 'dead', rule_version: 1 })
|
|
// Locked isn't junk: kept.
|
|
const e = await release()
|
|
const locked = await set('e', e, 2, null)
|
|
await db.table('release_verdicts').insert({ release_id: e, rule: 'locked', rule_version: 1 })
|
|
|
|
// Final: its holder held by a run of done ranges with hours to spare, nothing hashed (a readable upload's files).
|
|
const f = await release()
|
|
const final = await set('f', f, 2, null)
|
|
const posted = new Date('2024-03-01T12:00:00Z')
|
|
const [group] = await db.table('groups').insert({ name: 'alt.binaries.pruned', priority: 1 }).returning('id')
|
|
const groupId = Number(group.id)
|
|
// Finished (the window closed on it) long ago; its range done since.
|
|
await db.from('releases').where('id', f).update({ group_id: groupId, posted_at: posted, last_posted_at: posted, first_article: 5000, last_article: 5100, settled_at: old })
|
|
const [range] = await db
|
|
.table('ranges')
|
|
.insert({ group_id: groupId, start_id: 1, end_id: 10000, next_id: 10001, status: 'done', done_at: old, posted_low: new Date('2024-02-28T00:00:00Z'), posted_high: new Date('2024-03-03T00:00:00Z') })
|
|
.returning('id')
|
|
// A lone candidate unprobed since before the range was done: its hash may still match. Kept.
|
|
await db.table('probe_candidates').insert({ id: 9001, release_id: b, kind: 'lone', message_id: '<9001@x>', file: 'x', status: 'waiting', created_at: new Date(old.getTime() - 3600_000) })
|
|
|
|
// Out of budget: one batch of two sets (a, b), the next round goes on.
|
|
assert.deepEqual(await new Par2Pruner(2).round(0), { complete: 1, judged: 0, final: 0, rows: 2 })
|
|
assert.deepEqual(await new Par2Pruner(2).round(), { complete: 0, judged: 1, final: 0, rows: 2 })
|
|
assert.equal(await count(complete), 0)
|
|
assert.equal(await count(apart), 2)
|
|
assert.equal(await count(recent), 2)
|
|
assert.equal(await count(dead), 0)
|
|
assert.equal(await count(locked), 2)
|
|
assert.equal(await count(final), 2)
|
|
|
|
// Probed: final. The next pass starts over from the first set.
|
|
await db.from('probe_candidates').where('id', 9001).update({ status: 'done' })
|
|
assert.deepEqual(await new Par2Pruner(2).round(), { complete: 0, judged: 0, final: 1, rows: 2 })
|
|
assert.equal(await count(final), 0)
|
|
assert.equal((await db.from('par2_sets').count('* as n'))[0].n, '6')
|
|
|
|
// A hash of its files since the merger's last window: the merger hasn't matched it yet. Kept.
|
|
const g = await release()
|
|
const unseen = await set('g', g, 1, null)
|
|
await db.from('releases').where('id', g).update({ group_id: groupId, posted_at: posted, last_posted_at: posted, first_article: 6000, last_article: 6100, settled_at: old })
|
|
await db.table('worker_cursors').insert({ name: 'merger', at: new Date(Date.now() - 3600_000) })
|
|
await db.table('probe_candidates').insert({ id: 9002, release_id: await release(), kind: 'lone', message_id: '<9002@x>', file: 'x', status: 'done' })
|
|
await db.table('file_hashes').insert({ candidate_id: 9002, release_id: 1, hash16k: 'g0'.padEnd(32, '0'), size: 1000 })
|
|
assert.equal((await new Par2Pruner().round()).final, 0)
|
|
// Past the hash by more than the merger's overlap.
|
|
await db.from('worker_cursors').where('name', 'merger').update({ at: new Date(Date.now() + 120_000) })
|
|
assert.equal((await new Par2Pruner().round()).final, 1)
|
|
assert.equal(await count(unseen), 0)
|
|
assert.isNotNull(range)
|
|
})
|
|
|
|
test('sets of one created_at, a batch each: every one looked at', async ({ assert }) => {
|
|
// One prober flush gives thousands of sets the same created_at (to the microsecond).
|
|
const flush = new Date('2026-10-15T14:38:39.883Z')
|
|
let candidate = 7000
|
|
for (const letter of ['p', 'q', 'r']) {
|
|
const key = `pr${letter}`.padEnd(40, '0')
|
|
const row = { id: await insertRelease({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 1, readable: false }) }
|
|
const id = Number(row.id)
|
|
const setId = letter.repeat(32)
|
|
const h = `${letter}0`.padEnd(32, '0')
|
|
await db.table('par2_files').insert({ set_id: setId, file_id: h, hash16k: h, length: 1000, name: 'f.rar' })
|
|
candidate++
|
|
await db.table('probe_candidates').insert({ id: candidate, release_id: id, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
|
|
await db.table('file_hashes').insert({ candidate_id: candidate, release_id: id, hash16k: h, size: 1000 })
|
|
await db.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files, created_at) VALUES (?, ?, 1, ?::timestamptz + interval '984 microseconds')`, [setId, id, flush])
|
|
}
|
|
const round = await new Par2Pruner(1).round()
|
|
assert.equal(round.complete, 3)
|
|
assert.equal((await db.from('par2_files').whereIn('set_id', ['p'.repeat(32), 'q'.repeat(32), 'r'.repeat(32)]).count('* as n'))[0].n, '0')
|
|
// The pass ended: the next one starts from the first set.
|
|
assert.equal((await db.from('worker_cursors').where('name', 'par2-prune').firstOrFail()).key, '')
|
|
})
|
|
})
|