voyager 3: the batch store writes releases once, by id references and bytea keys
Each release chunk is a release inserted once: release_key the SHA-1 of the identity, the owner group and the first article (deterministic; no numbering under a group lock), identity_key the identity's SHA-1 bytes, poster_id and group_set_id through catalog_refs (cached per process), source as a code (weak-stem added), the name into release_texts. A finished release's candidates are claimable at once; a provisional tail release is finished by a later chunk of its key, a finished one sent again is left alone. Gone: joins into earlier releases and across groups, key numbering, the group lock, the fragment merger. Interim: chunks, counts and samples keyed by ranges.block_id, a range without a block gets one. Readers follow: search (posters and group_sets joined, words through release_texts, posters by id, source by code), the par2 merger, the sidecar folder, the judge's scraped rule, the spot inspector, the spotter, the NZB service, releases:regroup, edge:bench. Tests insert fixtures through tests/functional/helpers/releases.ts; the suites pass again. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc4e17e86f
commit
2580f20a43
32 files changed
+962
-497
No files matched your search
@@ -3,6 +3,7 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import { ReleaseJudge } from '#services/release_judge'
|
||||
import SearchService from '#services/search_service'
|
||||
import { insertRelease } from '#tests/functional/helpers/releases'
|
||||
|
||||
test.group('ReleaseJudge', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
@@ -15,9 +16,7 @@ test.group('ReleaseJudge', (group) => {
|
||||
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
||||
n++
|
||||
const key = `jj${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
const row = { id: await insertRelease({
|
||||
release_key: key,
|
||||
identity_key: key,
|
||||
name,
|
||||
@@ -29,8 +28,7 @@ test.group('ReleaseJudge', (group) => {
|
||||
readable: true,
|
||||
settled_at: settled,
|
||||
...extra,
|
||||
})
|
||||
.returning('id')
|
||||
}) }
|
||||
return Number(row.id)
|
||||
}
|
||||
let candidate = 9000
|
||||
@@ -93,10 +91,6 @@ test.group('ReleaseJudge', (group) => {
|
||||
poster: 'celeborn@not.here (Celeborn)',
|
||||
source: 'subject',
|
||||
})
|
||||
// Scraped posters are found through posters (the batch store adds each release's).
|
||||
await db
|
||||
.table('posters')
|
||||
.multiInsert([{ poster: '@_CelebMovie_@' }, { poster: 'celeborn@not.here (Celeborn)' }])
|
||||
const locked = await release('Locked.Thing', { passworded: true })
|
||||
|
||||
const unlocked = await release('Known.Password', { passworded: true })
|
||||
@@ -104,30 +98,19 @@ test.group('ReleaseJudge', (group) => {
|
||||
.table('release_passwords')
|
||||
.insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })
|
||||
|
||||
// A block of done ranges around the unusable ones (release_final); one beside an unread range isn't judged yet.
|
||||
// Unusable once finished (the window closed on it); a provisional tail release isn't judged yet.
|
||||
await db
|
||||
.table('groups')
|
||||
.insert({ id: 1, name: 'alt.binaries.judged', priority: 1 })
|
||||
.onConflict('id')
|
||||
.ignore()
|
||||
await db.table('ranges').multiInsert([
|
||||
{
|
||||
group_id: 1,
|
||||
start_id: 1,
|
||||
end_id: 1000,
|
||||
next_id: 1001,
|
||||
status: 'done',
|
||||
posted_low: new Date(settled.getTime() - 86400_000),
|
||||
posted_high: new Date(settled.getTime() + 86400_000),
|
||||
},
|
||||
{ group_id: 1, start_id: 1001, end_id: 2000, next_id: 1001, status: 'planned' },
|
||||
])
|
||||
const placed = { first_article: 100, last_article: 110 }
|
||||
const unusable = await release('ab12CD34ef56GH78ij90', { readable: false, ...placed })
|
||||
const pending = await release('pp12CD34ef56GH78ij90', {
|
||||
readable: false,
|
||||
first_article: 990,
|
||||
last_article: 1000,
|
||||
settled_at: null,
|
||||
posted_at: new Date(settled.getTime() + 86400_000 - 3600_000),
|
||||
})
|
||||
const fresh = await release('zz12CD34ef56GH78ij90', {
|
||||
@@ -188,9 +171,7 @@ test.group('ReleaseJudge', (group) => {
|
||||
const release = async (name: string, extra: Record<string, unknown> = {}) => {
|
||||
n++
|
||||
const key = `jc${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
const row = { id: await insertRelease({
|
||||
release_key: key,
|
||||
identity_key: key,
|
||||
name,
|
||||
@@ -201,8 +182,7 @@ test.group('ReleaseJudge', (group) => {
|
||||
readable: true,
|
||||
settled_at: settled,
|
||||
...extra,
|
||||
})
|
||||
.returning('id')
|
||||
}) }
|
||||
return Number(row.id)
|
||||
}
|
||||
const judge = new ReleaseJudge()
|
||||
|
||||
Reference in new issue
Block a user