Files
usenet-indexer/tests/functional/spot_listing.spec.ts
T
Daan MeijerandClaude Opus 5.5 be8a3d4580 Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29):
- Index free.pt fully and import spots with their descriptions, categories
  and NZBs; a spot renames its release, since Spotnet names are more
  trustworthy than names derived from subjects.
- Find passwords in spot descriptions too, and tell "needs a password we
  have" from "needs a password we don't have".

Content inspection:
- List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads
  and other archive-like files; mark yEncBin .bin containers as encrypted.
- Take real names from yEnc headers for randomly named files, and merge
  uploads that were split across posts.

Storage (D31, D32):
- Store segment lists zstd-compressed behind the FileStore accessor.
- group:backfill indexes older headers at low priority and pauses when
  the NVMe or HDD part of the database gets too large.

Grouping (D35, D36):
- Lone header segments wait in staged_segments until a second segment of
  their file arrives, which drops the per-article junk binaries.
- Counter grouping joins random-subject uploads by poster, [i/M] counter,
  part count and day.

Search and maintenance (D33, D34):
- A trigram-indexed search_text column, smaller indexes, estimated stats.
- Passes that clean up nonsense names; weekly index maintenance; GC budgets
  per kind.
- Web UI: grey out stale results while a search runs and cancel superseded
  requests. Count only up to 1,000 matches for the page, which made filter
  clicks take seconds (Newznab keeps the 10,000 cap).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 19:59:23 +02:00

110 lines
6.3 KiB
TypeScript

import { test } from '@japa/runner'
import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import SpotService, { isDummyNzb } from '#services/SpotService'
import SearchService from '#services/SearchService'
test.group('SpotService listings', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('old verified spots become visible releases without files', async ({ assert }) => {
const now = new Date()
const old = new Date('2019-05-01T12:00:00Z')
const spot = {
message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
}
await db.table('spots').insert(spot)
const [claimed] = await SpotService.claim(10)
assert.isTrue(SpotService.isListable(claimed))
assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
assert.equal(await SpotService.list([claimed]), 1)
const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
assert.equal(row.status, 'listed')
const release = await db.from('releases').where('id', row.release_id).first()
assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
assert.equal(Number(release.size), 1_500_000_000)
// The key an import of this spot's NZB uses, so it fills this release.
assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
})
test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
// A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
const overviews = Array.from({ length: 6000 }, (_, i) => [
i + 1,
{ 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
]) as any
assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
assert.equal(Number(count), 6000)
})
test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
const spot = {
message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
details_at: new Date(),
}
await db.table('spots').insert(spot)
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
const release = (await SearchService.release(Number(releaseId)))!
assert.equal(release.spot!.description, 'Plot[br]more')
assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
assert.equal(release.spot!.categories.head, 'Image')
assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
})
test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
// Text-only spots announce the dummy's own size.
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
})
})
test.group('SpotService passwords', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
const now = new Date()
await db.table('spots').insert({
message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
})
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
await db.from('releases').where('id', releaseId).update({ encrypted: true })
const release = (await SearchService.release(Number(releaseId)))!
assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
// An NZB's own password is not replaced.
await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
await SpotService.applyPassword('pw1@spot.net')
assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
})
test('password states', async ({ assert }) => {
const { passwordState } = await import('#services/SearchService')
assert.equal(passwordState(true, false), 'missing')
assert.equal(passwordState(true, true), 'known')
assert.equal(passwordState(null, true), 'known')
assert.equal(passwordState(false, false), 'none')
assert.equal(passwordState(null, false), 'none')
})
})