Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
6.3 KiB
TypeScript
110 lines
6.3 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { createHash } from 'node:crypto'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import SpotService, { isDummyNzb } from '#services/SpotService'
|
|
import SearchService from '#services/SearchService'
|
|
|
|
test.group('SpotService listings', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('old verified spots become visible releases without files', async ({ assert }) => {
|
|
const now = new Date()
|
|
const old = new Date('2019-05-01T12:00:00Z')
|
|
const spot = {
|
|
message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
|
|
poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
|
|
}
|
|
await db.table('spots').insert(spot)
|
|
const [claimed] = await SpotService.claim(10)
|
|
assert.isTrue(SpotService.isListable(claimed))
|
|
assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
|
|
assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
|
|
|
|
assert.equal(await SpotService.list([claimed]), 1)
|
|
const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
|
|
assert.equal(row.status, 'listed')
|
|
const release = await db.from('releases').where('id', row.release_id).first()
|
|
assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
|
|
assert.equal(Number(release.size), 1_500_000_000)
|
|
// The key an import of this spot's NZB uses, so it fills this release.
|
|
assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
|
|
|
|
const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
|
|
assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
|
|
})
|
|
|
|
test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
|
|
// A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
|
|
const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
|
|
const overviews = Array.from({ length: 6000 }, (_, i) => [
|
|
i + 1,
|
|
{ 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
|
|
]) as any
|
|
assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
|
|
const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
|
|
assert.equal(Number(count), 6000)
|
|
})
|
|
|
|
test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
|
|
const spot = {
|
|
message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
|
|
poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
|
|
created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
|
|
description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
|
|
details_at: new Date(),
|
|
}
|
|
await db.table('spots').insert(spot)
|
|
const [claimed] = await SpotService.claim(10)
|
|
await SpotService.list([claimed])
|
|
const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
|
|
const release = (await SearchService.release(Number(releaseId)))!
|
|
assert.equal(release.spot!.description, 'Plot[br]more')
|
|
assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
|
|
assert.equal(release.spot!.categories.head, 'Image')
|
|
assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
|
|
})
|
|
|
|
test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
|
|
const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
|
|
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
|
|
assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
|
|
// Text-only spots announce the dummy's own size.
|
|
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
|
|
})
|
|
})
|
|
|
|
test.group('SpotService passwords', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
|
|
const now = new Date()
|
|
await db.table('spots').insert({
|
|
message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
|
|
key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
|
|
description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
|
|
})
|
|
const [claimed] = await SpotService.claim(10)
|
|
await SpotService.list([claimed])
|
|
assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
|
|
const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
|
|
await db.from('releases').where('id', releaseId).update({ encrypted: true })
|
|
const release = (await SearchService.release(Number(releaseId)))!
|
|
assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
|
|
|
|
// An NZB's own password is not replaced.
|
|
await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
|
|
await SpotService.applyPassword('pw1@spot.net')
|
|
assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
|
|
})
|
|
|
|
test('password states', async ({ assert }) => {
|
|
const { passwordState } = await import('#services/SearchService')
|
|
assert.equal(passwordState(true, false), 'missing')
|
|
assert.equal(passwordState(true, true), 'known')
|
|
assert.equal(passwordState(null, true), 'known')
|
|
assert.equal(passwordState(false, false), 'none')
|
|
assert.equal(passwordState(null, false), 'none')
|
|
})
|
|
})
|