Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
bcb9be5da9
commit
be8a3d4580
97 files changed
+4454
-268
No files matched your search
+7
-1
@@ -42,7 +42,13 @@ export const plugins: Config['plugins'] = [
|
||||
*/
|
||||
export const runnerHooks: Required<Pick<Config, 'setup' | 'teardown'>> = {
|
||||
setup: [],
|
||||
teardown: [],
|
||||
teardown: [
|
||||
// Tests that complete a file queue jobs; their Redis connections would keep the process alive.
|
||||
async () => {
|
||||
const { default: QueueService } = await import('#services/QueueService')
|
||||
await QueueService.closeAll()
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
|
||||
test.group('BackfillService', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('queues header-only ranges downwards, capped, without holding back the live watermark', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
const batch = Number(indexerConfig.batchSize)
|
||||
const [g] = await db
|
||||
.table('groups')
|
||||
.insert({
|
||||
name: 'alt.binaries.backfilltest', active: true, fetch_bodies: true, last_queued_id: 5_000_000, last_indexed_id: 5_000_000,
|
||||
backfill_until_id: 5_000_000 - 100 * batch, backfill_next_id: 4_900_000, created_at: now, updated_at: now,
|
||||
})
|
||||
.returning('*')
|
||||
|
||||
const queued = await BackfillService.tick()
|
||||
assert.equal(queued, indexerConfig.backfill.maxQueued)
|
||||
const ranges = await db.from('fetch_ranges').where('group_id', g.id).orderBy('end_id', 'desc')
|
||||
assert.isTrue(ranges.every((r) => r.backfill && r.status === 'queued'))
|
||||
assert.equal(Number(ranges[0].end_id), 4_900_000)
|
||||
assert.equal(Number(ranges[0].start_id), 4_900_000 - batch + 1)
|
||||
const after = await db.from('groups').where('id', g.id).first()
|
||||
assert.equal(Number(after.backfill_next_id), 4_900_000 - queued * batch)
|
||||
|
||||
// The cap holds while those wait.
|
||||
assert.equal(await BackfillService.tick(), 0)
|
||||
|
||||
// Backfill ranges don't hold back the live watermark.
|
||||
await db.from('groups').where('id', g.id).update({ last_queued_id: 5_100_000 })
|
||||
assert.equal(await FetchRangeService.advanceWatermark({ id: g.id, name: g.name }), 5_100_000n)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,70 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import StagingService from '#services/StagingService'
|
||||
import IngestService, { segmentFromHeader } from '#services/IngestService'
|
||||
import { counterFileKey } from '#lib/subject/index'
|
||||
|
||||
let n = 0
|
||||
/** An article of a random-subject upload: a new random name every time. */
|
||||
const randomArticle = (k: number, total = 4, { counter = '[21/62]', date = 'Thu, 01 Oct 2026 12:00:00 GMT', id }: { counter?: string; date?: string; id?: string } = {}) =>
|
||||
segmentFromHeader(
|
||||
{ 'subject': `${counter} "Rnd${++n}xQ7kLm2Pz9Wv4Ty8Ab3Cd" yEnc (${k}/${total})`, 'from': 'bob@home.com', 'message-id': `<${id ?? `art${n}`}@nyuu>`, 'date': date, ':bytes': '700000' },
|
||||
'alt.binaries.boneless',
|
||||
k
|
||||
)!
|
||||
|
||||
async function ingestBatch(segments: ReturnType<typeof randomArticle>[], counters = true) {
|
||||
const r = await StagingService.filter(segments, { counters })
|
||||
await IngestService.ingest(r.ingest)
|
||||
return r
|
||||
}
|
||||
|
||||
test.group('Counter grouping', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('articles with random subjects become one file by poster, counter and part count', async ({ assert }) => {
|
||||
const key = counterFileKey(randomArticle(1))!
|
||||
await ingestBatch([randomArticle(1)])
|
||||
assert.isNull(await db.from('binaries').where('file_key', key).first())
|
||||
await ingestBatch([randomArticle(2)])
|
||||
const binary = await db.from('binaries').where('file_key', key).first()
|
||||
assert.isNotNull(binary)
|
||||
// Later articles join the file directly, until it completes.
|
||||
await ingestBatch([randomArticle(3)])
|
||||
await ingestBatch([randomArticle(4)])
|
||||
const parts = await db.from('parts').where('binary_id', binary.id).orderBy('number').select('number')
|
||||
assert.deepEqual(parts.map((p) => p.number), [1, 2, 3, 4])
|
||||
assert.isNotNull((await db.from('binaries').where('id', binary.id).first()).completed_at)
|
||||
})
|
||||
|
||||
test('off by default; another day is another file; a repeated part number is a conflict', async ({ assert }) => {
|
||||
await ingestBatch([randomArticle(1, 9, { counter: '[05/10]' })], false)
|
||||
await ingestBatch([randomArticle(2, 9, { counter: '[05/10]' })], false)
|
||||
assert.isNull(await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[05/10]' }))!).first())
|
||||
|
||||
const other = randomArticle(1, 9, { counter: '[06/10]', date: 'Fri, 02 Oct 2026 12:00:00 GMT' })
|
||||
assert.notEqual(counterFileKey(other), counterFileKey(randomArticle(1, 9, { counter: '[06/10]' })))
|
||||
|
||||
// Two uploads of the same shape on one day: part 1 twice, from different articles.
|
||||
await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
|
||||
await ingestBatch([randomArticle(2, 9, { counter: '[07/10]' })])
|
||||
await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
|
||||
const binary = await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[07/10]' }))!).first()
|
||||
assert.isNotNull(binary.conflicted_at)
|
||||
})
|
||||
|
||||
test('a normal file with a counter stays a normal file', async ({ assert }) => {
|
||||
const normal = (k: number) =>
|
||||
segmentFromHeader(
|
||||
{ 'subject': `[07/45] - "Some.Release.part06.rar" yEnc (${k}/3)`, 'from': 'poster@x', 'message-id': `<norm${k}@x>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
|
||||
'alt.binaries.test',
|
||||
k
|
||||
)!
|
||||
await ingestBatch([normal(1)])
|
||||
await ingestBatch([normal(2), normal(3)])
|
||||
const binary = await db.from('binaries').where('name', 'Some.Release.part06.rar').first()
|
||||
assert.equal(binary.file_key, normal(1).fileKey)
|
||||
assert.isNotNull(binary.completed_at)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import NameService from '#services/NameService'
|
||||
|
||||
@@ -36,10 +37,9 @@ test.group('NameService', (group) => {
|
||||
.table('releases')
|
||||
.insert({ release_key: 'w'.repeat(40), name: 'mOzyyxCyYUjAbCdEf12', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
const [file] = await db
|
||||
.table('files')
|
||||
.insert({ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', message_ids: '{}', release_id: weak.id, created_at: now, updated_at: now })
|
||||
.returning('id')
|
||||
const [file] = await FileStore.insert([
|
||||
{ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], release_id: weak.id, created_at: now, updated_at: now },
|
||||
])
|
||||
const episode = await binary('ep18', { size: 939_684_767, md5_16k: 'd'.repeat(32), release_id: weak.id, file_id: file.id })
|
||||
|
||||
await NameService.resolve([episode])
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import env from '#start/env'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
|
||||
@@ -11,11 +12,11 @@ async function seed(name: string, opts: { fileTotal?: number; files?: number } =
|
||||
.insert({ release_key: `nz${++counter}`.padEnd(40, 'n'), name, source: 'stem', poster: 'poster@example.invalid', groups: '["alt.binaries.test"]', file_total: opts.fileTotal ?? null, posted_at: now, first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
for (let i = 0; i < (opts.files ?? 1); i++) {
|
||||
await db.table('files').insert({
|
||||
await FileStore.insert([{
|
||||
filename: `${name}.part${i + 1}.rar`, poster: 'poster@example.invalid', date: now.getTime(), parts: 1, size: 1_000_000,
|
||||
release_id: release.id, groups: '["alt.binaries.test"]', message_ids: JSON.stringify({ '1': { id: `${name}-${i}@test`, size: 1_000_000 } }),
|
||||
release_id: release.id, groups: '["alt.binaries.test"]', segments: [{ number: 1, id: `${name}-${i}@test`, size: 1_000_000 }],
|
||||
created_at: now, updated_at: now,
|
||||
})
|
||||
}])
|
||||
}
|
||||
return Number(release.id)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import NzbImportService, { nzbFileName } from '#services/NzbImportService'
|
||||
import SearchService from '#services/SearchService'
|
||||
@@ -17,13 +19,10 @@ const XML = `<?xml version="1.0" encoding="utf-8"?>
|
||||
</file>
|
||||
</nzb>`
|
||||
|
||||
async function file(fields: Record<string, unknown>) {
|
||||
async function file(fields: Record<string, unknown> & { segments: Segment[] }) {
|
||||
const now = new Date()
|
||||
const [row] = await db
|
||||
.table('files')
|
||||
.insert({ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields })
|
||||
.returning('*')
|
||||
return row
|
||||
const [{ id }] = await FileStore.insert([{ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }])
|
||||
return db.from('files').where('id', id).first()
|
||||
}
|
||||
|
||||
test.group('NzbImportService', (group) => {
|
||||
@@ -32,6 +31,14 @@ test.group('NzbImportService', (group) => {
|
||||
test('names entries despite odd quoting', ({ assert }) => {
|
||||
assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar')
|
||||
assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2')
|
||||
// Spot NZBs without quotes.
|
||||
assert.equal(nzbFileName('Adventure.part0055.rar - [056/175] (1/1)'), 'Adventure.part0055.rar')
|
||||
assert.equal(nzbFileName('LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2 (1/0) (1/0)'), 'LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2')
|
||||
assert.equal(nzbFileName('Reacher.S04.vol001+02.par2 yEnc'), 'Reacher.S04.vol001+02.par2')
|
||||
assert.equal(nzbFileName('06 - A New Land, a New Music.flac Mehmet Ali Sanlikol - The Rise Up (2020) (01/35)'), '06 - A New Land, a New Music.flac')
|
||||
assert.equal(nzbFileName('[01/10] - Some.Release.2026.part01.rar yEnc (1/50) 35000000'), 'Some.Release.2026.part01.rar')
|
||||
assert.equal(nzbFileName('Mozart Zauberfloete 2026 mp4 720p.zip.001'), 'Mozart Zauberfloete 2026 mp4 720p.zip.001')
|
||||
assert.equal(nzbFileName('LibreOffice_7.6.4_Win_x86-64.msi.vol0000'), 'LibreOffice_7.6.4_Win_x86-64.msi.vol0000')
|
||||
})
|
||||
|
||||
test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => {
|
||||
@@ -40,8 +47,8 @@ test.group('NzbImportService', (group) => {
|
||||
.table('releases')
|
||||
.insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
|
||||
const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', message_ids: JSON.stringify({ '1': { id: 'nzb-1@x', size: 2000 } }) })
|
||||
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
|
||||
const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', segments: [{ number: 1, id: 'nzb-1@x', size: 2000 }] })
|
||||
|
||||
const result = (await NzbImportService.import(posted, XML))!
|
||||
assert.equal(result.releaseId, Number(release.id))
|
||||
@@ -70,7 +77,7 @@ test.group('NzbImportService', (group) => {
|
||||
<groups><group>alt.binaries.x</group></groups>
|
||||
<segments><segment bytes="500" number="1">${id}@x</segment></segments>
|
||||
</file></nzb>`
|
||||
const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', message_ids: JSON.stringify({ '1': { id: 'rar-1@x', size: 2000 } }) })
|
||||
const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', segments: [{ number: 1, id: 'rar-1@x', size: 2000 }] })
|
||||
|
||||
const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))!
|
||||
const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))!
|
||||
@@ -87,11 +94,44 @@ test.group('NzbImportService', (group) => {
|
||||
(await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id
|
||||
const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT')
|
||||
const lonely = await insert('lonely', '.nzb')
|
||||
await file({ filename: 'x.part1.rar', release_id: target, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) })
|
||||
const posted = await file({ filename: '.nzb', release_id: lonely, message_ids: JSON.stringify({ '1': { id: 'nzb-2@x', size: 2000 } }) })
|
||||
await file({ filename: 'x.part1.rar', release_id: target, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
|
||||
const posted = await file({ filename: '.nzb', release_id: lonely, segments: [{ number: 1, id: 'nzb-2@x', size: 2000 }] })
|
||||
|
||||
await NzbImportService.import(posted, XML)
|
||||
assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target))
|
||||
assert.notExists(await db.from('releases').where('id', lonely).first())
|
||||
})
|
||||
|
||||
test('a trusted NZB (a spot) takes its files into its own release, named after it', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
const insertRelease = async (key: string, name: string, source: string) =>
|
||||
(
|
||||
await db
|
||||
.table('releases')
|
||||
.insert({ release_key: key.repeat(40).slice(0, 40), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
)[0].id
|
||||
// Name rules had put the spot's first volume into a season release with another episode.
|
||||
const season = await insertRelease('s', 'zxcvvbgnhgjytygf', 'stem')
|
||||
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: season, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
|
||||
await file({ filename: 'other.episode.rar', release_id: season, segments: [{ number: 1, id: 'other-1@x', size: 700 }] })
|
||||
const spotFile = { id: null, filename: 'x.nzb', poster: 'Bart', groups: [], date: now.getTime(), first_message_id: 'spot1@spot.net' }
|
||||
|
||||
const result = (await NzbImportService.import(spotFile, XML, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', moveContainer: false, trusted: true }))!
|
||||
assert.notEqual(result.releaseId, Number(season))
|
||||
const own = await db.from('releases').where('id', result.releaseId).first()
|
||||
assert.include(own, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', file_count: 2 })
|
||||
const left = await db.from('files').where('release_id', season).select('filename')
|
||||
assert.deepEqual(left.map((f) => f.filename), ['other.episode.rar'])
|
||||
|
||||
// A par2 or archive name no longer replaces a spot's title.
|
||||
const { isBetterName } = await import('#services/NameService')
|
||||
assert.isTrue(isBetterName('zxcvvbgnhgjytygf', 'Better.Name.S03E285'))
|
||||
|
||||
// Another spot for the same files (a repost) joins the first one's release instead of taking them.
|
||||
const repost = { ...spotFile, first_message_id: 'spot2@spot.net' }
|
||||
const second = (await NzbImportService.import(repost, XML, { name: 'Repost', source: 'spot', moveContainer: false, trusted: true }))!
|
||||
assert.equal(second.releaseId, result.releaseId)
|
||||
assert.equal(second.releaseName, 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,27 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
|
||||
// Names seen on homelab (LOG.md 2026-10-01 (3)).
|
||||
const RANDOM = [
|
||||
'5d4063b423f57e1eadda1068b0091409', 'mOzyyxCyYUjAbCdEf12', '6327d9fa2d6acf3e9a61fb053359d5e7.1790574045',
|
||||
'e63afdb4-6a1f-417d-aa1b-b14c0ea1b81d', '1650777167', '83111763 n', 'prvjr7hapd0i1ny14pz4', '8huowjcdamwbetufib9f',
|
||||
'071vl0l120ndtjkjkuh34a2g6aqsak750f55dkudeaea8s4bivjm1l32fd3uduk0dab2t82hp553i29nfjvjt3redp', 'wHJ7s71sAGE', 'EmFqTfL8rxV',
|
||||
'VIDEO_TS', 'Sample', 'CD1', 'Disc 2', 'Subs',
|
||||
]
|
||||
const READABLE = [
|
||||
'Our.America.with.Lisa.Ling.2011.S01E01.Faith.Healers.1080p.DSCP.WEB-DL.DDP2.0.H.264-FUZEER', '00-graham_barham-club_country',
|
||||
'1790274935_nicovideo_jp_watch_sm26602185', 'StarWars1977', 'nyuu_live_roundtrip', 'Makaja_Gonzales-Freestyle-WEB-2021-XTC_iNT',
|
||||
'The Tom and Jerry Show (2014) S02E13 - Slap Happy Birthday', 'Bondgenoten', 'S01E02', '2026', 'Samples.Of.Love.2020.1080p', 'Subs.and.Dubs',
|
||||
]
|
||||
|
||||
test.group('random names', () => {
|
||||
test('looksRandom and SQL is_readable_name agree', async ({ assert }) => {
|
||||
for (const name of [...RANDOM, ...READABLE]) {
|
||||
const { rows } = await db.rawQuery('SELECT is_readable_name(?) AS readable', [name])
|
||||
const random = RANDOM.includes(name)
|
||||
assert.equal(looksRandom(name), random, `looksRandom(${name})`)
|
||||
assert.equal(rows[0].readable, !random, `is_readable_name(${name})`)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,42 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import FileStore from '#services/FileStore'
|
||||
import ReleaseMergeService from '#services/ReleaseMergeService'
|
||||
|
||||
let keys = 0
|
||||
async function postRelease(name: string, files: string[], poster: string) {
|
||||
const now = new Date()
|
||||
const [r] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: `merge${++keys}`.padEnd(40, 'm'), name, source: 'post', poster, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
await FileStore.insert(
|
||||
files.map((f, i) => ({ filename: f, poster, date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: '[]',
|
||||
segments: [{ number: 1, id: `${name}-${poster}-${i}@x`, size: 10 }], created_at: now, updated_at: now }))
|
||||
)
|
||||
return Number(r.id)
|
||||
}
|
||||
|
||||
test.group('Split uploads', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('merges one upload posted under a poster per volume, not reposts or other uploads', async ({ assert }) => {
|
||||
const name = 'Masters.of.the.Universe.1987.Theatrical.Cut.2160p.BDRip-GRP'
|
||||
const a = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part001.rar'], 'niSSYNy5zOW2VX@VMuLu_e7Gr.bIx')
|
||||
const b = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part002.rar'], 'MgrZzFLB3D@MeK4_gntYXzh9g.LBQ')
|
||||
const c = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part003.rar'], 'qNGVviOMN2AJrSzeXFZqvW@VzavS9GO.Y0b')
|
||||
// A repost: the same file names again.
|
||||
const repostName = 'Some.Repost.2026.1080p-GRP'
|
||||
const r1 = await postRelease(repostName, ['abc.part01.rar'], 'p1@x')
|
||||
const r2 = await postRelease(repostName, ['abc.part01.rar'], 'p2@x')
|
||||
|
||||
assert.deepEqual(await ReleaseMergeService.mergeSplitPosts({ dryRun: true }), { names: 1, releases: 2 })
|
||||
await ReleaseMergeService.mergeSplitPosts()
|
||||
assert.equal((await db.from('releases').where('id', a).first()).file_count, 3)
|
||||
assert.equal((await db.from('releases').where('id', b).first()).file_count, 0)
|
||||
assert.equal((await db.from('releases').where('id', c).first()).file_count, 0)
|
||||
assert.equal((await db.from('releases').where('id', r1).first()).file_count, 1)
|
||||
assert.equal((await db.from('releases').where('id', r2).first()).file_count, 1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import FileStore from '#services/FileStore'
|
||||
import SearchService, { searchNormalize } from '#services/SearchService'
|
||||
|
||||
test.group('Search text', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('normalises like SQL does', async ({ assert }) => {
|
||||
assert.equal(searchNormalize(' The.Block.AU_S22E35 - 1080p '), 'the block au s22e35 1080p')
|
||||
const { rows } = await db.rawQuery(`SELECT search_normalize(' The.Block.AU_S22E35 - 1080p ') AS n`)
|
||||
assert.equal(rows[0].n, 'the block au s22e35 1080p')
|
||||
})
|
||||
|
||||
test('follows files and renames, only for visible releases', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
const [r] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: 's'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7Rt', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('*')
|
||||
assert.isNull(r.search_text)
|
||||
|
||||
// A readable file name that the release name lacks joins the search text.
|
||||
await FileStore.insert([
|
||||
{ filename: 'Some.Documentary.2026.1080p.WEB.part01.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
|
||||
groups: '[]', segments: [{ number: 1, id: 'st1@x', size: 10 }], created_at: now, updated_at: now },
|
||||
{ filename: 'Q7wErTyUiOpAsDfGhJkL9.part02.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
|
||||
groups: '[]', segments: [{ number: 1, id: 'st2@x', size: 10 }], created_at: now, updated_at: now },
|
||||
])
|
||||
let row = await db.from('releases').where('id', r.id).first()
|
||||
assert.equal(row.search_text, 'xq8r2kdlm3zw91pqa7rt some documentary 2026 1080p web')
|
||||
const found = await SearchService.search({ q: 'documentary 2026.1080p' } as any)
|
||||
assert.isTrue(found.releases.some((x: any) => x.id === Number(r.id)))
|
||||
|
||||
// A rename recomputes it.
|
||||
await db.from('releases').where('id', r.id).update({ name: 'Some.Documentary.2026.1080p.WEB-GRP' })
|
||||
row = await db.from('releases').where('id', r.id).first()
|
||||
assert.equal(row.search_text, 'some documentary 2026 1080p web grp some documentary 2026 1080p web')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,59 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import FileStore from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import { parseSfv } from '#lib/content/index'
|
||||
|
||||
const POSTER = 'Snazzy Poster <me@localhost.localdomain>'
|
||||
|
||||
let keys = 0
|
||||
async function release(name: string, source = 'stem') {
|
||||
const now = new Date()
|
||||
const [r] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: `sfv${++keys}`.padEnd(40, 'x'), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
return Number(r.id)
|
||||
}
|
||||
|
||||
async function file(name: string, releaseId: number, poster = POSTER) {
|
||||
const [{ id }] = await FileStore.insert([
|
||||
{ filename: name, poster, date: Date.now(), parts: 1, size: 10, release_id: releaseId, groups: '[]',
|
||||
segments: [{ number: 1, id: `${name}@x`, size: 10 }], created_at: new Date(), updated_at: new Date() },
|
||||
])
|
||||
return db.from('files').where('id', id).first()
|
||||
}
|
||||
|
||||
test.group('SFV grouping', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('parses SFV listings', ({ assert }) => {
|
||||
assert.deepEqual(parseSfv('; made by x\r\n01-a-b-4066667a.flac 4066667A\r\n"02 c.flac" 5058F120\r\n\r\n'), [
|
||||
{ name: '01-a-b-4066667a.flac', crc: '4066667a' },
|
||||
{ name: '02 c.flac', crc: '5058f120' },
|
||||
])
|
||||
})
|
||||
|
||||
test("gathers an album's tracks into the SFV's release, named after the album; later tracks join too", async ({ assert }) => {
|
||||
const album = await release('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026')
|
||||
const sfv = await file('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026.sfv', album)
|
||||
const track1 = await release('01-graham_barham-country_music-4066667a')
|
||||
await file('01-graham_barham-country_music-4066667a.flac', track1)
|
||||
// Another poster's file with the same name stays.
|
||||
const other = await release('01-graham_barham-country_music-4066667a', 'stem')
|
||||
const foreign = await file('01-graham_barham-country_music-4066667a.flac', other, 'someone@else')
|
||||
|
||||
const listing = '01-graham_barham-country_music-4066667a.flac 4066667a\n02-graham_barham-sippin-5058f120.flac 5058f120\n'
|
||||
assert.equal(await SfvService.attach(sfv, listing), 1)
|
||||
const renamed = await db.from('releases').where('id', album).first()
|
||||
assert.equal(renamed.name, 'graham_barham-club_country_(after_party)-16bit-web-flac-2026')
|
||||
assert.equal((await db.from('files').where('id', foreign.id).first()).release_id, other)
|
||||
|
||||
// Track 2 completes after the SFV was read.
|
||||
const track2 = await release('02-graham_barham-sippin-5058f120')
|
||||
const late = await file('02-graham_barham-sippin-5058f120.flac', track2)
|
||||
assert.equal(await SfvService.claim({ id: late.id, name: late.filename, poster: late.poster, date: Number(late.date), release_id: track2 }), album)
|
||||
assert.equal((await db.from('releases').where('id', album).first()).file_count, 3)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,109 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import SpotService, { isDummyNzb } from '#services/SpotService'
|
||||
import SearchService from '#services/SearchService'
|
||||
|
||||
test.group('SpotService listings', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('old verified spots become visible releases without files', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
const old = new Date('2019-05-01T12:00:00Z')
|
||||
const spot = {
|
||||
message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
|
||||
poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
|
||||
}
|
||||
await db.table('spots').insert(spot)
|
||||
const [claimed] = await SpotService.claim(10)
|
||||
assert.isTrue(SpotService.isListable(claimed))
|
||||
assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
|
||||
assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
|
||||
|
||||
assert.equal(await SpotService.list([claimed]), 1)
|
||||
const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
|
||||
assert.equal(row.status, 'listed')
|
||||
const release = await db.from('releases').where('id', row.release_id).first()
|
||||
assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
|
||||
assert.equal(Number(release.size), 1_500_000_000)
|
||||
// The key an import of this spot's NZB uses, so it fills this release.
|
||||
assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
|
||||
|
||||
const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
|
||||
assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
|
||||
})
|
||||
|
||||
test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
|
||||
// A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
|
||||
const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
|
||||
const overviews = Array.from({ length: 6000 }, (_, i) => [
|
||||
i + 1,
|
||||
{ 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
|
||||
]) as any
|
||||
assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
|
||||
const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
|
||||
assert.equal(Number(count), 6000)
|
||||
})
|
||||
|
||||
test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
|
||||
const spot = {
|
||||
message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
|
||||
poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
|
||||
created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
|
||||
description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
|
||||
details_at: new Date(),
|
||||
}
|
||||
await db.table('spots').insert(spot)
|
||||
const [claimed] = await SpotService.claim(10)
|
||||
await SpotService.list([claimed])
|
||||
const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
|
||||
const release = (await SearchService.release(Number(releaseId)))!
|
||||
assert.equal(release.spot!.description, 'Plot[br]more')
|
||||
assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
|
||||
assert.equal(release.spot!.categories.head, 'Image')
|
||||
assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
|
||||
})
|
||||
|
||||
test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
|
||||
const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
|
||||
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
|
||||
assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
|
||||
// Text-only spots announce the dummy's own size.
|
||||
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('SpotService passwords', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
|
||||
const now = new Date()
|
||||
await db.table('spots').insert({
|
||||
message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
|
||||
key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
|
||||
description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
|
||||
})
|
||||
const [claimed] = await SpotService.claim(10)
|
||||
await SpotService.list([claimed])
|
||||
assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
|
||||
const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
|
||||
await db.from('releases').where('id', releaseId).update({ encrypted: true })
|
||||
const release = (await SearchService.release(Number(releaseId)))!
|
||||
assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
|
||||
|
||||
// An NZB's own password is not replaced.
|
||||
await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
|
||||
await SpotService.applyPassword('pw1@spot.net')
|
||||
assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
|
||||
})
|
||||
|
||||
test('password states', async ({ assert }) => {
|
||||
const { passwordState } = await import('#services/SearchService')
|
||||
assert.equal(passwordState(true, false), 'missing')
|
||||
assert.equal(passwordState(true, true), 'known')
|
||||
assert.equal(passwordState(null, true), 'known')
|
||||
assert.equal(passwordState(false, false), 'none')
|
||||
assert.equal(passwordState(null, false), 'none')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,70 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import StagingService from '#services/StagingService'
|
||||
import IngestService, { segmentFromHeader } from '#services/IngestService'
|
||||
|
||||
const segment = (name: string, k: number, n: number, id = `${name}-${k}@x`) =>
|
||||
segmentFromHeader(
|
||||
{ 'subject': `"${name}" yEnc (${k}/${n})`, 'from': 'poster@x', 'message-id': `<${id}>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
|
||||
'alt.binaries.test',
|
||||
k
|
||||
)!
|
||||
|
||||
async function binary(name: string) {
|
||||
return db.from('binaries').where('name', name).first()
|
||||
}
|
||||
|
||||
test.group('Staging', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('a lone segment waits; its second segment brings both in', async ({ assert }) => {
|
||||
let r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
|
||||
assert.equal(r.staged, 1)
|
||||
assert.lengthOf(r.ingest, 0)
|
||||
await IngestService.ingest(r.ingest)
|
||||
assert.isNull(await binary('rnd1.bin'))
|
||||
|
||||
// The same article again (a re-fetched range) is no second segment.
|
||||
r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
|
||||
assert.lengthOf(r.ingest, 0)
|
||||
|
||||
// Nor is the same article crossposted to another group (other group, other article number).
|
||||
const crosspost = { ...segment('rnd1.bin', 3, 50), groups: ['alt.binaries.boneless'], articleNumber: 160154327571 }
|
||||
r = await StagingService.filter([crosspost])
|
||||
assert.lengthOf(r.ingest, 0)
|
||||
r = await StagingService.filter([segment('dup.bin', 2, 9), { ...segment('dup.bin', 2, 9), groups: ['alt.binaries.boneless'] }])
|
||||
assert.equal(r.staged, 1)
|
||||
|
||||
r = await StagingService.filter([segment('rnd1.bin', 4, 50)])
|
||||
assert.deepEqual(r.ingest.map((s) => s.number).sort(), [3, 4])
|
||||
await IngestService.ingest(r.ingest)
|
||||
const b = await binary('rnd1.bin')
|
||||
assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 2)
|
||||
assert.lengthOf(await db.from('staged_segments').where('file_key', segment('rnd1.bin', 3, 50).fileKey), 0)
|
||||
|
||||
// Later segments of a known file go straight in.
|
||||
r = await StagingService.filter([segment('rnd1.bin', 9, 50)])
|
||||
assert.lengthOf(r.ingest, 1)
|
||||
})
|
||||
|
||||
test('single-part files and files with two segments in the batch go straight in; staged ones join them', async ({ assert }) => {
|
||||
// Part 1 arrived alone at the end of the previous batch.
|
||||
await StagingService.filter([segment('real.part01.rar', 1, 3)])
|
||||
const r = await StagingService.filter([segment('real.part01.rar', 2, 3), segment('real.part01.rar', 3, 3), segment('one.nfo', 1, 1)])
|
||||
assert.equal(r.staged, 0)
|
||||
assert.sameMembers(r.ingest.map((s) => `${s.name}:${s.number}`), ['real.part01.rar:1', 'real.part01.rar:2', 'real.part01.rar:3', 'one.nfo:1'])
|
||||
})
|
||||
|
||||
test('GC adopts segments whose file got a binary, and drops old ones', async ({ assert }) => {
|
||||
await StagingService.filter([segment('late.bin', 7, 10)])
|
||||
await StagingService.filter([segment('old.bin', 2, 10)])
|
||||
await db.from('staged_segments').where('file_key', segment('old.bin', 2, 10).fileKey).update({ staged_at: new Date(Date.now() - 3 * 3600_000) })
|
||||
// Another job created the binary meanwhile.
|
||||
await IngestService.ingest([segment('late.bin', 1, 10), segment('late.bin', 2, 10)])
|
||||
const result = await StagingService.sweep(2 * 3600, Date.now() + 10_000)
|
||||
assert.deepEqual(result, { adopted: 1, expired: 1 })
|
||||
const b = await binary('late.bin')
|
||||
assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 3)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
|
||||
async function seed() {
|
||||
@@ -8,12 +9,12 @@ async function seed() {
|
||||
.table('releases')
|
||||
.insert({ release_key: 'w'.repeat(40), name: 'Webtest.Show.S01E02.1080p', source: 'par2', groups: '["alt.binaries.test"]', posted_at: now, first_seen_at: now, last_seen_at: now, contents: JSON.stringify([{ name: 'Webtest.Show.S01E02.1080p/episode.mkv', size: 1000 }]) })
|
||||
.returning('id')
|
||||
await db.table('files').insert({
|
||||
await FileStore.insert([{
|
||||
filename: 'Xq8r2KdLm3Zw91PqA7webtest', real_name: 'Webtest.Show.S01E02.1080p.mkv', poster: 'p@example.invalid',
|
||||
date: now.getTime(), parts: 2, size: 1400, release_id: release.id, groups: '["alt.binaries.test"]',
|
||||
message_ids: JSON.stringify({ '1': { id: 'one@test', size: 700 }, '2': { id: 'two@test', size: 700 } }),
|
||||
segments: [{ number: 1, id: 'one@test', size: 700 }, { number: 2, id: 'two@test', size: 700 }],
|
||||
created_at: now, updated_at: now,
|
||||
})
|
||||
}])
|
||||
return Number(release.id)
|
||||
}
|
||||
|
||||
@@ -67,21 +68,21 @@ test.group('Web API', (group) => {
|
||||
const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete
|
||||
const file = (releaseId: number, name: string) => ({
|
||||
filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
|
||||
groups: '[]', message_ids: '{}', created_at: now, updated_at: now,
|
||||
groups: '[]', segments: [], created_at: now, updated_at: now,
|
||||
})
|
||||
|
||||
// Counter rule: [i/M] announced 2 files.
|
||||
const [counted] = await db.table('releases').insert({ release_key: 'c'.repeat(40), name: 'Counted', groups: '[]', file_total: 2, first_seen_at: now, last_seen_at: now }).returning('id')
|
||||
await db.table('files').insert(file(counted.id, 'a.rar'))
|
||||
await FileStore.insert([file(counted.id, 'a.rar')])
|
||||
assert.isFalse(await complete(counted.id))
|
||||
await db.table('files').insert(file(counted.id, 'b.rar'))
|
||||
await FileStore.insert([file(counted.id, 'b.rar')])
|
||||
assert.isTrue(await complete(counted.id))
|
||||
|
||||
// par2 rule: the listed file becomes present once a file of its exact size is there.
|
||||
const [listed] = await db.table('releases').insert({ release_key: 'l'.repeat(40), name: 'Listed', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id')
|
||||
await db.table('release_files').insert({ release_id: listed.id, par2_file_id: '1'.repeat(32), name: 'Listed.mkv', size: 5_000_000, md5_16k: '0'.repeat(32) })
|
||||
assert.isFalse(await complete(listed.id))
|
||||
const [f] = await db.table('files').insert(file(listed.id, 'Xq8r2KdLm3Zw91PqA7')).returning('id')
|
||||
const [f] = await FileStore.insert([file(listed.id, 'Xq8r2KdLm3Zw91PqA7')])
|
||||
const [b] = await db.table('binaries').insert({ file_key: 'z'.repeat(40), poster: 'p', subject: '"x" yEnc', name: 'x', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, file_id: f.id }).returning('id')
|
||||
assert.isFalse(await complete(listed.id))
|
||||
await db.from('binaries').where('id', b.id).update({ size: 5_000_000 })
|
||||
@@ -94,8 +95,8 @@ test.group('Web API', (group) => {
|
||||
.insert({ release_key: 'p'.repeat(40), name: 'Webtest.Par2only.S01E01.1080p.WEB-DL-GRP', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
await db.table('release_files').insert({ release_id: release.id, par2_file_id: '2'.repeat(32), name: 'Webtest.Par2only.S01E01.mkv', size: 1_000_000_000, md5_16k: '0'.repeat(32) })
|
||||
await db.table('files').insert({ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
|
||||
release_id: release.id, groups: '[]', message_ids: '{}', created_at: now, updated_at: now })
|
||||
await FileStore.insert([{ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
|
||||
release_id: release.id, groups: '[]', segments: [], created_at: now, updated_at: now }])
|
||||
|
||||
const hidden = await client.get('/api/v1/releases').qs({ q: 'webtest par2only' })
|
||||
assert.notInclude(hidden.body().releases.map((r: any) => r.id), Number(release.id))
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { test } from '@japa/runner'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import FileStore from '#services/FileStore'
|
||||
import ContentService from '#services/ContentService'
|
||||
|
||||
let keys = 0
|
||||
async function counterFile(name: string) {
|
||||
const now = new Date()
|
||||
const [r] = await db
|
||||
.table('releases')
|
||||
.insert({ release_key: `yenc${++keys}`.padEnd(40, 'y'), name, source: 'weak-stem', groups: '[]', first_seen_at: now, last_seen_at: now })
|
||||
.returning('id')
|
||||
const [{ id }] = await FileStore.insert([
|
||||
{ filename: name, poster: 'bob@home.com', date: now.getTime(), parts: 2, size: 10, release_id: r.id, groups: '["alt.binaries.boneless"]',
|
||||
segments: [{ number: 1, id: `${name}@nyuu`, size: 10 }], created_at: now, updated_at: now },
|
||||
])
|
||||
return db.from('files').where('id', id).first()
|
||||
}
|
||||
|
||||
test.group('Names from the yEnc header', (group) => {
|
||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||
|
||||
test('random-named files take the yEnc name and gather in its release', async ({ assert }) => {
|
||||
const a = await counterFile('P9BSPl59iDfQUVvXFeBVG4MDQaOTufds')
|
||||
const b = await counterFile('Faoq6nlEaVlTzYyNA27sSfFm2KvLHOvD')
|
||||
const nameFromYenc = (ContentService as any).nameFromYenc.bind(ContentService)
|
||||
assert.equal(await nameFromYenc(a, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar'), 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar')
|
||||
await nameFromYenc(b, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part02.rar')
|
||||
const [fa, fb] = await Promise.all([db.from('files').where('id', a.id).first(), db.from('files').where('id', b.id).first()])
|
||||
assert.equal(fa.release_id, fb.release_id)
|
||||
assert.equal((await db.from('releases').where('id', fa.release_id).first()).name, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP')
|
||||
|
||||
// A random yEnc name, or a file that has a real name already, stays as it is.
|
||||
const c = await counterFile('Vg9QflfNLXVMPOWtK4KRJv474DadgTIF')
|
||||
assert.isUndefined(await nameFromYenc(c, null, 'qTcrIl1qTo4Bx6El5c4RtTA4ZSqwd6tz'))
|
||||
const d = await counterFile('Some.Release.part01.rar')
|
||||
assert.isUndefined(await nameFromYenc(d, null, 'Other.Name.part01.rar'))
|
||||
})
|
||||
})
|
||||
@@ -30,6 +30,15 @@ test.group('content / sniffContentType', () => {
|
||||
})
|
||||
|
||||
test.group('yenc / lenientSize', () => {
|
||||
test("recognises yEncBin's encrypted .bin container by its header", ({ assert }) => {
|
||||
// The first 48 bytes of 2141406641.bin (yEncBin@Poster.com).
|
||||
const data = Buffer.from('54f399d117c492696e4dc4ef5b60c11d7f83e92dfdf23450e79370b048123da81e4c8013091b682fb22478daed2ae981', 'hex')
|
||||
assert.equal(sniffContentType(data), 'yencbin')
|
||||
// The second variant (1060238685.bin).
|
||||
assert.equal(sniffContentType(Buffer.from('54f399e917c492696a4e9fb05b60c31d7f83e92df9f16f5086e3c3d9480d66ef1e48d915', 'hex')), 'yencbin')
|
||||
assert.isNull(sniffContentType(Buffer.from('54f399d117c492696e4dc4ef5b60c11e7f83e92d', 'hex')))
|
||||
})
|
||||
|
||||
test('decodes a part whose declared size leaves out a header, still checking the CRC', ({ assert }) => {
|
||||
const data = Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(16, 7), Buffer.from('ciphertext-here')])
|
||||
const declared = data.length - 20
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
|
||||
test.group('nntp / articleAtDate', () => {
|
||||
test('finds the article posted at a time, skipping gaps', async ({ assert }) => {
|
||||
// One article a second from t0, articles 1,000,000–1,100,000; 1,040,000–1,040,999 expired.
|
||||
const t0 = Date.UTC(2026, 8, 1)
|
||||
const dateOf = async (n: bigint) => (n >= 1_040_000n && n < 1_041_000n ? null : t0 + Number(n - 1_000_000n) * 1000)
|
||||
const at = await articleAtDate(1_000_000n, 1_100_000n, t0 + 60_000 * 1000, dateOf, 100n)
|
||||
assert.isAtLeast(Number(at), 1_059_900)
|
||||
assert.isAtMost(Number(at), 1_060_000)
|
||||
assert.equal(await articleAtDate(1_000_000n, 1_100_000n, t0 - 1, dateOf, 100n), 1_000_000n)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { delimitedReader } from '#lib/nntp/delimited_reader'
|
||||
|
||||
const CRLF = Buffer.from('\r\n')
|
||||
const END = Buffer.from('\r\n.\r\n')
|
||||
|
||||
async function* chunks(...parts: string[]) {
|
||||
for (const part of parts) yield Buffer.from(part, 'latin1')
|
||||
}
|
||||
|
||||
test.group('nntp / delimitedReader', () => {
|
||||
test('reads an empty multi-line reply (XOVER of a range without articles)', async ({ assert }) => {
|
||||
const reader = delimitedReader(chunks('224 Overview follows\r\n.\r\n', '211 1 2 3 g\r\n'), CRLF)
|
||||
assert.equal((await reader.next()).value!.toString(), '224 Overview follows')
|
||||
assert.equal((await reader.next(END)).value!.toString(), '')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '211 1 2 3 g')
|
||||
})
|
||||
|
||||
test('reads multi-line data, also with the terminator split over chunks', async ({ assert }) => {
|
||||
for (const split of [1, 2, 3, 4, 5, 6, 7]) {
|
||||
const reply = '224 ok\r\nline 1\r\n..dotted\r\nline 3\r\n.\r\n200 next\r\n'
|
||||
const at = reply.indexOf('.\r\n200') - 2 + split - 1
|
||||
const reader = delimitedReader(chunks(reply.slice(0, at), reply.slice(at)), CRLF)
|
||||
assert.equal((await reader.next()).value!.toString(), '224 ok')
|
||||
assert.equal((await reader.next(END)).value!.toString(), 'line 1\r\n..dotted\r\nline 3', `split ${split}`)
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '200 next')
|
||||
}
|
||||
})
|
||||
|
||||
test('reads replies byte by byte', async ({ assert }) => {
|
||||
const text = '200 welcome\r\n224 ok\r\na\r\n.\r\n224 ok\r\n.\r\n211 x\r\n'
|
||||
const reader = delimitedReader(chunks(...text.split('')), CRLF)
|
||||
assert.equal((await reader.next()).value!.toString(), '200 welcome')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
|
||||
assert.equal((await reader.next(END)).value!.toString(), 'a')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
|
||||
assert.equal((await reader.next(END)).value!.toString(), '')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '211 x')
|
||||
})
|
||||
|
||||
test('keeps binary data intact', async ({ assert }) => {
|
||||
const body = Buffer.from([0, 13, 255, 10, 46, 13, 46, 10, 13])
|
||||
const source = (async function* () {
|
||||
yield Buffer.from('222 body\r\n')
|
||||
yield body
|
||||
yield Buffer.from('\r\n.\r\n')
|
||||
})()
|
||||
const reader = delimitedReader(source, CRLF)
|
||||
await reader.next()
|
||||
assert.deepEqual((await reader.next(END)).value, body)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,46 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { zstdCompressSync, constants } from 'node:zlib'
|
||||
import { NICOVIDEO_FILE, nicovideoInfo, nicovideoReleaseName, zstdPrefix } from '#lib/content/index'
|
||||
|
||||
function tarEntry(name: string, data: Buffer): Buffer {
|
||||
const header = Buffer.alloc(512)
|
||||
header.write(name, 0, 'utf8')
|
||||
header.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 'latin1')
|
||||
header.write('ustar\0', 257, 'latin1')
|
||||
return Buffer.concat([header, data, Buffer.alloc((512 - (data.length % 512)) % 512)])
|
||||
}
|
||||
|
||||
test.group('nicovideo captures', () => {
|
||||
test('reads the title from the start of a capture, even a truncated one', async ({ assert }) => {
|
||||
const dir = '1790720489_nicovideo_jp_watch_sm27965722'
|
||||
const response = {
|
||||
data: {
|
||||
metadata: { jsonLds: [{ author: { name: 'まふまふ' } }] },
|
||||
response: {
|
||||
video: { id: 'sm27965722', title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】', registeredAt: '2016-01-08T20:00:00+09:00' },
|
||||
owner: { nickname: 'まふまふ' },
|
||||
tag: { items: [{ name: '音楽' }, { name: 'After_the_Rain' }] },
|
||||
},
|
||||
},
|
||||
}
|
||||
const tar = Buffer.concat([
|
||||
tarEntry(`${dir}/`, Buffer.alloc(0)),
|
||||
tarEntry(`${dir}/server_response.metadata`, Buffer.from(JSON.stringify(response, null, 2))),
|
||||
tarEntry(`${dir}/01.cmfa`, randomBytes(300_000)),
|
||||
])
|
||||
const compressed = zstdCompressSync(tar, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
|
||||
// Only the first part of the stream, as in the file's first article.
|
||||
const info = nicovideoInfo(await zstdPrefix(compressed.subarray(0, Math.floor(compressed.length * 0.6))))!
|
||||
assert.deepEqual(info, {
|
||||
id: 'sm27965722',
|
||||
title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】',
|
||||
owner: 'まふまふ',
|
||||
registeredAt: '2016-01-08T20:00:00+09:00',
|
||||
tags: ['音楽', 'After_the_Rain'],
|
||||
})
|
||||
assert.equal(nicovideoReleaseName(info, 'sm1'), '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】 - まふまふ (nicovideo sm27965722)')
|
||||
assert.equal(NICOVIDEO_FILE.exec(`${dir}.tar.zst`)![1], 'sm27965722')
|
||||
assert.isNull(nicovideoInfo(await zstdPrefix(Buffer.from('not zstd'))))
|
||||
})
|
||||
})
|
||||
+13
-1
@@ -10,7 +10,7 @@ test.group('nzb / buildNzb', () => {
|
||||
poster: 'poster@example.invalid',
|
||||
date: 1_790_000_000_000,
|
||||
groups: ['alt.binaries.test'],
|
||||
segments: { '2': { id: 'b@x', size: 700 }, '1': { id: 'a@x', size: 800 } },
|
||||
segments: [{ number: 2, id: 'b@x', size: 700 }, { number: 1, id: 'a@x', size: 800 }],
|
||||
},
|
||||
],
|
||||
{ title: 'Some.Show.S01E01' }
|
||||
@@ -53,3 +53,15 @@ test.group('nzb / parseNzb', () => {
|
||||
assert.isNull(parseNzb('<html>not an nzb</html>'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('nzb / stored segment lists', () => {
|
||||
test('pack and unpack round-trip, in order, also with gaps in the numbering', async ({ assert }) => {
|
||||
const { packSegments, unpackSegments } = await import('#lib/nzb/segments')
|
||||
const sequential = Array.from({ length: 300 }, (_, i) => ({ number: i + 1, id: `Ab${i}Cd-1790600369${i}@nyuu`, size: i === 299 ? 1234 : 740000 }))
|
||||
assert.deepEqual(unpackSegments(packSegments([...sequential].reverse())), sequential)
|
||||
const gaps = [{ number: 3, id: 'c@x', size: 3 }, { number: 1, id: 'a@x', size: 1 }, { number: 7, id: 'g@x', size: 7 }]
|
||||
assert.deepEqual(unpackSegments(packSegments(gaps)), [gaps[1], gaps[0], gaps[2]])
|
||||
assert.deepEqual(unpackSegments(packSegments([])), [])
|
||||
assert.isBelow(packSegments(sequential).length / sequential.length, 30)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,71 @@
|
||||
import { test } from '@japa/runner'
|
||||
import {
|
||||
checkHeader,
|
||||
listingStart,
|
||||
parseBsdtarListing,
|
||||
parseSevenZipStart,
|
||||
readNumber,
|
||||
sevenZipVolume,
|
||||
} from '#lib/sevenzip/index'
|
||||
|
||||
// `7z a tiny.7z Tiny.Release` (Tiny.Release/a.txt): an LZMA-encoded header.
|
||||
const ENCODED = Buffer.from(
|
||||
'N3q8ryccAATeIVLCfQAAAAAAAAAhAAAAAAAAAGkLW/MBAAtoZWxsbyB3b3JsZAoAAACBMweuD87ysgwHyEN/QbH6/duIttdfpOXRhZbi6t7tycGOUDyW1l2aTE7xmuZ/NAXKgg599VxKYAF1XImsfT9LzHQKaEH6PEZambLnwJM62mXXEJIpaYvhCrHFeQv+jLnYjgJMfwZEcAAAABcGEAEJbQAHCwEAASMDAQEFXQAQAAAMgJYKASTou9wAAA==',
|
||||
'base64'
|
||||
)
|
||||
// The same with -mhc=off: a plain header.
|
||||
const PLAIN = Buffer.from(
|
||||
'N3q8ryccAASpRwRsEAAAAAAAAACWAAAAAAAAACTou9wBAAtoZWxsbyB3b3JsZAoAAQQGAAEJEAAHCwEAASEhAQAMDAAICgEtOwivAAAFAg4BgBkJAAAAAAAAAAAAEUEAVABpAG4AeQAuAFIAZQBsAGUAYQBzAGUAAABUAGkAbgB5AC4AUgBlAGwAZQBhAHMAZQAvAGEALgB0AHgAdAAAABkCAAAUEgEA/QYYLBBR3QH9BhgsEFHdARUKAQAQgP1BIIC0gQAA',
|
||||
'base64'
|
||||
)
|
||||
|
||||
test.group('sevenzip', () => {
|
||||
test('finds the header at the end from the signature header', ({ assert }) => {
|
||||
const start = parseSevenZipStart(ENCODED)!
|
||||
assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })
|
||||
const header = ENCODED.subarray(start.headerStart, start.archiveSize)
|
||||
assert.isTrue(checkHeader(start, header))
|
||||
assert.isFalse(checkHeader(start, Buffer.alloc(33)))
|
||||
// Encoded: the packed header right after the file data (packPos 16) is needed too.
|
||||
assert.equal(listingStart(start, header), 48)
|
||||
|
||||
const plain = parseSevenZipStart(PLAIN)!
|
||||
assert.equal(plain.archiveSize, PLAIN.length)
|
||||
assert.equal(listingStart(plain, PLAIN.subarray(plain.headerStart)), plain.headerStart)
|
||||
})
|
||||
|
||||
test('rejects other data and a damaged signature header', ({ assert }) => {
|
||||
assert.isNull(parseSevenZipStart(Buffer.from('Rar!\x1a\x07\x01\x00'.padEnd(40, '\0'), 'latin1')))
|
||||
const damaged = Buffer.from(ENCODED)
|
||||
damaged[14] ^= 1
|
||||
assert.isNull(parseSevenZipStart(damaged))
|
||||
})
|
||||
|
||||
test('reads 7z numbers', ({ assert }) => {
|
||||
assert.deepEqual(readNumber(Buffer.from([0x10]), 0), [16, 1])
|
||||
assert.deepEqual(readNumber(Buffer.from([0x81, 0x02]), 0), [0x102, 2])
|
||||
assert.deepEqual(readNumber(Buffer.from([0xc0, 0x34, 0x12]), 0), [0x1234, 3])
|
||||
assert.deepEqual(readNumber(Buffer.from([0xff, 1, 0, 0, 0, 0, 0, 0, 0]), 0), [1, 9])
|
||||
})
|
||||
|
||||
test('parses bsdtar listings', ({ assert }) => {
|
||||
const out = [
|
||||
'drwxrwxr-x 0 0 0 0 Sep 30 19:13 Some.Release.2026/',
|
||||
'-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release.2026/movie.mkv',
|
||||
'-rw-rw-r-- 0 0 0 3 Jan 1 2019 Some.Release.2026/some info.nfo',
|
||||
'',
|
||||
].join('\n')
|
||||
assert.deepEqual(parseBsdtarListing(out), [
|
||||
{ name: 'Some.Release.2026', size: 0, directory: true },
|
||||
{ name: 'Some.Release.2026/movie.mkv', size: 3000000, directory: false },
|
||||
{ name: 'Some.Release.2026/some info.nfo', size: 3, directory: false },
|
||||
])
|
||||
})
|
||||
|
||||
test('numbers volumes', ({ assert }) => {
|
||||
assert.equal(sevenZipVolume('x.7z.001'), 1)
|
||||
assert.equal(sevenZipVolume('X.7Z.012'), 12)
|
||||
assert.equal(sevenZipVolume('x.7z'), 0)
|
||||
assert.isNull(sevenZipVolume('x.part01.rar'))
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,15 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { deflateRawSync } from 'node:zlib'
|
||||
import { decodeSpotBinary, parseSpotArticleHeaders, parseSpotHeader, parseSpotXml, verifySpotHeader } from '#lib/spotnet/index'
|
||||
import {
|
||||
decodeSpotBinary,
|
||||
decodeSpotText,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
spotCategoryLabels,
|
||||
verifySpotHeader,
|
||||
} from '#lib/spotnet/index'
|
||||
import { ubbToText } from '#lib/newznab/xml'
|
||||
|
||||
// A real spot from free.pt (self-signed, key 7), 2026-09-28.
|
||||
const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "<jWCbaAa4f1A7LC6agmOEx@spot.net>", "date": "Mon, 28 Sep 2026 18:24:45 GMT"}
|
||||
@@ -44,6 +53,62 @@ test.group('spotnet / xml and nzb', () => {
|
||||
assert.deepEqual(spot.subcats, ['01a09', '01z00'])
|
||||
})
|
||||
|
||||
test('reads the description, tag and image', ({ assert }) => {
|
||||
const spot = parseSpotXml(
|
||||
`<Spotnet><Posting><Key>7</Key><Tag>Bart</Tag><Title>T</Title>` +
|
||||
`<Description>Lily Tomlin[br][br]If you're ‘in’ [url=https://example.com/a]here[/url]</Description>` +
|
||||
`<Image Width='185' Height='278'><Segment>FYe6l94kQsY7LC6agh5Xn@spot.net</Segment></Image>` +
|
||||
`<Category>01<Sub>01a09</Sub></Category></Posting></Spotnet>`
|
||||
)!
|
||||
assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]')
|
||||
assert.equal(spot.tag, 'Bart')
|
||||
assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 })
|
||||
// Older spots link an image on the web.
|
||||
assert.deepEqual(parseSpotXml('<Posting><Image>http://x.nl/a.jpg</Image></Posting>')!.image, { url: 'http://x.nl/a.jpg' })
|
||||
assert.isNull(parseSpotXml('<Posting><Title>T</Title></Posting>')!.image)
|
||||
})
|
||||
|
||||
test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => {
|
||||
const raw = [
|
||||
'From: vanker',
|
||||
' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>',
|
||||
'Subject: The postman | The postman |',
|
||||
'X-XML: <SpotNet><Posting><Title>The postman</Title><Description><![CDATA[post voor de lol.<br />Van "The',
|
||||
'X-XML: postman"]]></Description><Image>http://covertarget.com/si.php?id=1</Image><Category>00</Category></Posting></SpotNet>',
|
||||
'Date: 01 Feb 2010 02:43:16 GMT',
|
||||
].join('\r\n')
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw)
|
||||
assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>')
|
||||
assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT')
|
||||
const spot = parseSpotXml(xml)!
|
||||
assert.equal(spot.description, 'post voor de lol.<br />Van "The postman"')
|
||||
assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' })
|
||||
assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"')
|
||||
})
|
||||
|
||||
test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => {
|
||||
assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café')
|
||||
assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café')
|
||||
})
|
||||
|
||||
test('names categories as Spotweb does', ({ assert }) => {
|
||||
assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), {
|
||||
head: 'Image',
|
||||
subcats: [
|
||||
{ type: 'Type', values: ['Movie'] },
|
||||
{ type: 'Format', values: ['x264'] },
|
||||
{ type: 'Source', values: ['Retail'] },
|
||||
{ type: 'Language', values: ['English subtitles (available)', 'English audio/written'] },
|
||||
{ type: 'Genre', values: ['Science Fiction'] },
|
||||
],
|
||||
})
|
||||
assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] })
|
||||
})
|
||||
|
||||
test('turns UBB markup into plain text', ({ assert }) => {
|
||||
assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)')
|
||||
})
|
||||
|
||||
test('decodes an escaped, deflated NZB split over articles', ({ assert }) => {
|
||||
const nzb = '<?xml version="1.0"?>\n<nzb>\r\n=\0</nzb>'.repeat(20)
|
||||
const escaped = deflateRawSync(Buffer.from(nzb, 'latin1'))
|
||||
@@ -68,3 +133,33 @@ test.group('spotnet / moderation', () => {
|
||||
assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('spotnet / passwords in descriptions', () => {
|
||||
test('finds passwords written in the description or title', async ({ assert }) => {
|
||||
const { passwordFromSpot } = await import('#lib/spotnet/index')
|
||||
assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2')
|
||||
assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R')
|
||||
assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813')
|
||||
assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401')
|
||||
assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo')
|
||||
assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t')
|
||||
assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
|
||||
})
|
||||
|
||||
test('ignores sentences about passwords', async ({ assert }) => {
|
||||
const { passwordFromSpot } = await import('#lib/spotnet/index')
|
||||
for (const description of [
|
||||
'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]',
|
||||
'[b]--> Geen wachtwoord benodigd [/b]',
|
||||
'zitten 2 bestanden in en is met een password beschermd.',
|
||||
'It also removes the need for manual password entry and improves the reliability',
|
||||
'Codeermodus: Hoge kwaliteit (2-Pass VBR)',
|
||||
'29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper',
|
||||
'Wachtwoord: geen',
|
||||
'WW: zie nzb',
|
||||
'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
|
||||
]) {
|
||||
assert.isNull(passwordFromSpot('T', description), description)
|
||||
}
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user