daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: fresh AdonisJS 7 API app on Postgres

Browse Source 
The rebuild branch starts over (REBUILD.md): the API starter kit, with
Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a
shared one in development) and the #lib alias. Phase 1's code is
imported from the current app module by module.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:05:24 +02:00
1 parent 699395c51f
commit 96f0ade10e
264 files changed
+777 -24132
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/functional/backfill.spec.ts
-38
View File
@@ -1,38 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import indexerConfig from '#config/indexer'
import BackfillService from '#services/BackfillService'
import FetchRangeService from '#services/FetchRangeService'
test.group('BackfillService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('queues header-only ranges downwards, capped, without holding back the live watermark', async ({ assert }) => {
const now = new Date()
const batch = Number(indexerConfig.batchSize)
const [g] = await db
.table('groups')
.insert({
name: 'alt.binaries.backfilltest', active: true, fetch_bodies: true, last_queued_id: 5_000_000, last_indexed_id: 5_000_000,
backfill_until_id: 5_000_000 - 100 * batch, backfill_next_id: 4_900_000, created_at: now, updated_at: now,
})
.returning('*')
const queued = await BackfillService.tick()
assert.equal(queued, indexerConfig.backfill.maxQueued)
const ranges = await db.from('fetch_ranges').where('group_id', g.id).orderBy('end_id', 'desc')
assert.isTrue(ranges.every((r) => r.backfill && r.status === 'queued'))
assert.equal(Number(ranges[0].end_id), 4_900_000)
assert.equal(Number(ranges[0].start_id), 4_900_000 - batch + 1)
const after = await db.from('groups').where('id', g.id).first()
assert.equal(Number(after.backfill_next_id), 4_900_000 - queued * batch)
// The cap holds while those wait.
assert.equal(await BackfillService.tick(), 0)
// Backfill ranges don't hold back the live watermark.
await db.from('groups').where('id', g.id).update({ last_queued_id: 5_100_000 })
assert.equal(await FetchRangeService.advanceWatermark({ id: g.id, name: g.name }), 5_100_000n)
})
})
tests/functional/binary_files.spec.ts
-86
View File
@@ -1,86 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import BinaryFileService from '#services/BinaryFileService'
import QueueService from '#services/QueueService'
import FileStore, { segmentsOf } from '#services/FileStore'
import { fileKey } from '#lib/subject/index'
import ContentService from '#services/ContentService'
function segment(name: string, number: number, total = 3): IngestSegment {
const poster = 'batch-test@example.invalid'
const subject = `"${name}" yEnc`
return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] }
}
const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first()
test.group('Binaries to files', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => {
await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)])
// The same article again (a crosspost): not counted twice.
await IngestService.ingest([segment('a.bin', 2)])
assert.equal((await binary('a.bin')).part_count, 2)
assert.isNull((await binary('a.bin')).completed_at)
const completed = await IngestService.ingest([segment('a.bin', 3)])
assert.equal((await binary('a.bin')).part_count, 3)
assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey])
})
test('a binary from before the count is counted exactly on its next part', async ({ assert }) => {
await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)])
await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null })
const completed = await IngestService.ingest([segment('b.bin', 3)])
assert.equal((await binary('b.bin')).part_count, 3)
assert.lengthOf(completed, 1)
})
test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => {
await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)]))
await IngestService.ingest([segment('half.bin', 1)])
// known.bin is a file already (e.g. from an NZB), found by its first segment.
const now = new Date()
const [known] = await FileStore.insert([
{ filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now },
])
const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id))
const result = await BinaryFileService.store(ids)
assert.include(result, { saved: 1, existing: 1 })
assert.deepEqual(result.incomplete, [ids[2]])
const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first()
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id))
assert.equal(Number((await binary('known.bin')).file_id), known.id)
// Header parts are in the file's segment list now.
const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n')
assert.equal(Number(n), 0)
// The retry: done ones are skipped, the incomplete one again incomplete.
const again = await BinaryFileService.store(ids)
assert.include(again, { saved: 0, skipped: 2 })
assert.deepEqual(again.incomplete, [ids[2]])
})
test('a queued obfuscated file keeps its binary until inspection fingerprints it', async ({ assert }) => {
const name = 'Zk3m9QpL2xW8rT5vN1bYq.bin'
await IngestService.ingest([1, 2, 3].map((n) => segment(name, n)))
const b = await binary(name)
const result = await BinaryFileService.store([Number(b.id)])
assert.include(result, { saved: 1, inspections: 1 })
const waiting = await binary(name)
assert.isTrue(waiting.awaiting_inspection)
// Inspection stores the fingerprint and clears the flag.
const task = { kind: 'inspect' as const, fileId: Number(waiting.file_id), name, segments: [], sevenZip: null }
await ContentService.apply(task, { size: 300, contentType: null, md5_16k: 'a'.repeat(32), yencName: name })
const inspected = await binary(name)
assert.isNull(inspected.awaiting_inspection)
assert.equal(inspected.md5_16k, 'a'.repeat(32))
})
})
tests/functional/counter_grouping.spec.ts
-70
View File
@@ -1,70 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import StagingService from '#services/StagingService'
import IngestService, { segmentFromHeader } from '#services/IngestService'
import { counterFileKey } from '#lib/subject/index'
let n = 0
/** An article of a random-subject upload: a new random name every time. */
const randomArticle = (k: number, total = 4, { counter = '[21/62]', date = 'Thu, 01 Oct 2026 12:00:00 GMT', id }: { counter?: string; date?: string; id?: string } = {}) =>
segmentFromHeader(
{ 'subject': `${counter} "Rnd${++n}xQ7kLm2Pz9Wv4Ty8Ab3Cd" yEnc (${k}/${total})`, 'from': 'bob@home.com', 'message-id': `<${id ?? `art${n}`}@nyuu>`, 'date': date, ':bytes': '700000' },
'alt.binaries.boneless',
k
)!
async function ingestBatch(segments: ReturnType<typeof randomArticle>[], counters = true) {
const r = await StagingService.filter(segments, { counters })
await IngestService.ingest(r.ingest)
return r
}
test.group('Counter grouping', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('articles with random subjects become one file by poster, counter and part count', async ({ assert }) => {
const key = counterFileKey(randomArticle(1))!
await ingestBatch([randomArticle(1)])
assert.isNull(await db.from('binaries').where('file_key', key).first())
await ingestBatch([randomArticle(2)])
const binary = await db.from('binaries').where('file_key', key).first()
assert.isNotNull(binary)
// Later articles join the file directly, until it completes.
await ingestBatch([randomArticle(3)])
await ingestBatch([randomArticle(4)])
const parts = await db.from('parts').where('binary_id', binary.id).orderBy('number').select('number')
assert.deepEqual(parts.map((p) => p.number), [1, 2, 3, 4])
assert.isNotNull((await db.from('binaries').where('id', binary.id).first()).completed_at)
})
test('off by default; another day is another file; a repeated part number is a conflict', async ({ assert }) => {
await ingestBatch([randomArticle(1, 9, { counter: '[05/10]' })], false)
await ingestBatch([randomArticle(2, 9, { counter: '[05/10]' })], false)
assert.isNull(await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[05/10]' }))!).first())
const other = randomArticle(1, 9, { counter: '[06/10]', date: 'Fri, 02 Oct 2026 12:00:00 GMT' })
assert.notEqual(counterFileKey(other), counterFileKey(randomArticle(1, 9, { counter: '[06/10]' })))
// Two uploads of the same shape on one day: part 1 twice, from different articles.
await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
await ingestBatch([randomArticle(2, 9, { counter: '[07/10]' })])
await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
const binary = await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[07/10]' }))!).first()
assert.isNotNull(binary.conflicted_at)
})
test('a normal file with a counter stays a normal file', async ({ assert }) => {
const normal = (k: number) =>
segmentFromHeader(
{ 'subject': `[07/45] - "Some.Release.part06.rar" yEnc (${k}/3)`, 'from': 'poster@x', 'message-id': `<norm${k}@x>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
'alt.binaries.test',
k
)!
await ingestBatch([normal(1)])
await ingestBatch([normal(2), normal(3)])
const binary = await db.from('binaries').where('name', 'Some.Release.part06.rar').first()
assert.equal(binary.file_key, normal(1).fileKey)
assert.isNotNull(binary.completed_at)
})
})
tests/functional/deferred.spec.ts
-67
View File
@@ -1,67 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import DeferredService, { deferredName, isDeferred } from '#services/DeferredService'
const header = (id: string, name: string, extra: Record<string, string> = {}) => ({
'message-id': `<${id.padEnd(32, '0')}@ngPost>`,
'subject': `"${name}" yEnc`,
'from': 'pKeUimunULJZz@ngPost.com',
':bytes': '740000',
'date': 'Mon, 28 Sep 2026 10:00:00 +0000',
...extra,
})
test.group('DeferredService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('recognises deferred uploads by message-id and a quoted name', ({ assert }) => {
assert.equal(deferredName('[026/109] "HFIX7fbQAuH4PZsvGMw7Pz.part025.rar" yEnc'), 'HFIX7fbQAuH4PZsvGMw7Pz.part025.rar')
assert.isNull(deferredName('no quotes here'))
assert.isTrue(isDeferred(header('ab', 'x.part1.rar')))
assert.isFalse(isDeferred({ ...header('ab', 'x'), 'message-id': '<part1of5.abc@nyuu>' }))
})
test('articles without a name in the subject are kept as article ranges', async ({ assert }) => {
const hex = (id: string, n: number) => ({ articleNumber: n, header: { ...header(id, 'x'), subject: id.padEnd(32, 'f') } })
const files = await DeferredService.record('alt.binaries.test', [hex('a', 100), hex('b', 180), hex('c', 90_000), hex('d', 150)])
assert.equal(files, 0)
const ranges = await db.from('deferred_ranges').orderBy('first_article')
assert.deepEqual(
ranges.map((r) => [Number(r.first_article), Number(r.last_article), r.article_count]),
[[100, 180, 3], [90_000, 90_000, 1]]
)
assert.lengthOf(await db.from('deferred_files'), 0)
})
test('aggregates articles per file, across batches, counting only the first group', async ({ assert }) => {
await DeferredService.record('alt.binaries.test', [
{ articleNumber: 105, header: header('b', 'a.part1.rar') },
{ articleNumber: 101, header: header('a', 'a.part1.rar') },
{ articleNumber: 102, header: header('c', 'b.par2', { ':bytes': '1000' }) },
])
await DeferredService.record('alt.binaries.test', [{ articleNumber: 110, header: header('d', 'a.part1.rar') }])
// A crosspost seen in another group doesn't count twice.
await DeferredService.record('alt.binaries.misc', [{ articleNumber: 9, header: header('a', 'a.part1.rar') }])
const rows = await db.from('deferred_files').orderBy('name')
assert.lengthOf(rows, 2)
const [a, b] = rows
assert.include(a, { name: 'a.part1.rar', group_name: 'alt.binaries.test', article_count: 3, status: 'pending' })
assert.equal(Number(a.first_article), 101)
assert.equal(Number(a.last_article), 110)
assert.equal(a.first_message_id, 'a'.padEnd(32, '0') + '@ngPost')
assert.equal(Number(a.bytes), 3 * 740000)
assert.include(b, { name: 'b.par2', article_count: 1 })
})
test('only claims files that stopped growing', async ({ assert }) => {
await DeferredService.record('alt.binaries.test', [{ articleNumber: 1, header: header('a', 'new.rar') }])
assert.lengthOf(await DeferredService.claim(10), 0)
await db.from('deferred_files').update({ last_seen_at: new Date(Date.now() - 24 * 60 * 60 * 1000) })
const claimed = await DeferredService.claim(10)
assert.lengthOf(claimed, 1)
assert.equal(claimed[0].status, 'processing')
assert.lengthOf(await DeferredService.claim(10), 0)
})
})
tests/functional/fetch_ranges.spec.ts
-46
View File
@@ -1,46 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FetchRangeService from '#services/FetchRangeService'
async function groupWithRanges(ranges: [number, number, string][]) {
const now = new Date()
const [group] = await db
.table('groups')
.insert({ name: `test.ranges.${Date.now()}`, active: false, last_indexed_id: 100, last_queued_id: 700, created_at: now, updated_at: now })
.returning(['id', 'name'])
for (const [start, end, status] of ranges) {
await db.table('fetch_ranges').insert({ group_id: group.id, start_id: start, end_id: end, status, created_at: now, updated_at: now })
}
return { id: Number(group.id), name: group.name }
}
test.group('FetchRangeService.advanceWatermark', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('stops just below the first range that is not done', async ({ assert }) => {
const g = await groupWithRanges([
[101, 200, 'done'],
[201, 300, 'failed'],
[301, 400, 'done'],
[401, 700, 'queued'],
])
assert.equal(await FetchRangeService.advanceWatermark(g), 200n)
// done ranges are forgotten, the gap and the queued range stay
const left = await db.from('fetch_ranges').where('group_id', g.id).orderBy('start_id').select('status')
assert.deepEqual(left.map((r) => r.status), ['failed', 'queued'])
})
test('reaches the queue cursor when everything is done', async ({ assert }) => {
const g = await groupWithRanges([
[101, 400, 'done'],
[401, 700, 'done'],
])
assert.equal(await FetchRangeService.advanceWatermark(g), 700n)
})
test('never moves backwards', async ({ assert }) => {
const g = await groupWithRanges([[50, 60, 'failed']])
assert.equal(await FetchRangeService.advanceWatermark(g), 100n)
})
})
tests/functional/file_keys.spec.ts
-41
View File
@@ -1,41 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileKeys from '#services/FileKeys'
import FileStore from '#services/FileStore'
const file = (key: string, name: string) => {
const now = new Date()
return { file_key: key, filename: name, poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [], created_at: now, updated_at: now }
}
test.group('file_keys', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.each.teardown(() => FileKeys.forget())
test('the fill copies keys from before, then file inserts use the filter', async ({ assert }) => {
// A file from before file_keys: in files, not in file_keys.
const [old] = await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'old.bin')])
await db.rawQuery(`DELETE FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-old'.padEnd(40, '0')])
await db.from('file_keys_fill').where('id', 1).update({ next_id: 0, until_id: null, complete: false })
FileKeys.forget()
// Not complete: every partition is checked, so the old key is found anyway.
assert.isFalse(await FileKeys.complete())
assert.lengthOf(await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'again.bin')]), 0)
const filled = await FileKeys.fill(60_000)
assert.isTrue(filled.complete)
assert.isAtLeast(filled.untilId, old.id)
const [{ n }] = (await db.rawQuery(`SELECT count(*) AS n FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-old'.padEnd(40, '0')])).rows
assert.equal(Number(n), 1)
// Complete: the filter says the old key may exist, the partitions confirm it; a new key goes in.
assert.isTrue(await FileKeys.complete())
assert.lengthOf(await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'again.bin')]), 0)
assert.lengthOf(await FileStore.insert([file('fk-new'.padEnd(40, '0'), 'new.bin')]), 1)
// And the trigger put the new one in file_keys.
const [{ m }] = (await db.rawQuery(`SELECT count(*) AS m FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-new'.padEnd(40, '0')])).rows
assert.equal(Number(m), 1)
})
})
tests/functional/file_partitions.spec.ts
-51
View File
@@ -1,51 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import FilePartitions, { FILE_PARTITION_SIZE } from '#services/FilePartitions'
const file = (fileKey: string, name: string) => {
const now = new Date()
return { file_key: fileKey, filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], created_at: now, updated_at: now }
}
test.group('files partitions', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
// Sequences ignore rollbacks: put files_id_seq back after each test.
group.each.setup(async () => {
const { rows } = await db.rawQuery(`SELECT last_value, is_called FROM files_id_seq`)
return async () => {
await db.rawQuery(`SELECT setval('files_id_seq', ?, ?)`, [rows[0].last_value, rows[0].is_called])
}
})
test('a file_key already in an older partition is not inserted again', async ({ assert }) => {
await db.rawQuery(`SELECT setval('files_id_seq', 500000, false)`)
const [first] = await FileStore.insert([file('k'.repeat(40), 'old.bin')])
// The next ids fall in the second partition, whose own unique index doesn't know the key.
await db.rawQuery(`SELECT setval('files_id_seq', 1000000, false)`)
const again = await FileStore.insert([file('k'.repeat(40), 'again.bin'), file('n'.repeat(40), 'new.bin'), file('n'.repeat(40), 'twice.bin')])
assert.deepEqual(again.map((f) => f.filename), ['new.bin'])
assert.isAtLeast(again[0].id, 1000000)
assert.equal(await db.from('files').where('file_key', 'k'.repeat(40)).count('* as n').then((r) => Number(r[0].n)), 1)
assert.isBelow(first.id, 1000000)
})
test('partitions are kept ready ahead of the newest id', async ({ assert }) => {
const upper = async () => {
const { rows } = await db.rawQuery(
`SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass`
)
return Number(rows[0].upper)
}
await FilePartitions.ensure()
const before = await upper()
assert.deepEqual(await FilePartitions.ensure(), [])
// Ids handed out up to just inside the four partitions kept ahead: one more is needed.
await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - 4 * FILE_PARTITION_SIZE + 10])
await FileStore.insert([file('e'.repeat(40), 'late.bin')])
assert.deepEqual(await FilePartitions.ensure(), [`files_p${before}`])
assert.equal(await upper(), before + FILE_PARTITION_SIZE)
})
})
tests/functional/garbage.spec.ts
-190
View File
@@ -1,190 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import indexerConfig from '#config/indexer'
import FileStore from '#services/FileStore'
import SeedService from '#services/SeedService'
import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
import ContentService from '#services/ContentService'
import GarbageFilter from '#services/GarbageFilter'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toSeedFile } from '#lib/seed/wire'
import { explored, uploadTool } from '#lib/garbage/index'
const GROUP = 'alt.binaries.test'
let n = 0
/** Headers of a file posted by ngPost (or nyuu), all its parts. */
function upload(name: string, parts: number, tool: 'ngPost' | 'nyuu' = 'ngPost') {
const id = () => {
n++
return tool === 'ngPost' ? `${n.toString(16).padStart(32, '0')}@ngPost` : `${'FmFwIaTpGgUiSfYcMwFl'}${String.fromCharCode(65 + (n % 26))}e${String.fromCharCode(65 + ((n >> 5) % 26))}o-${String(1790637260000 + n)}@nyuu`
}
return Array.from({ length: parts }, (_, i) =>
segmentFromHeader(
{
'subject': `[1/1] - "${name}" yEnc (${i + 1}/${parts})`,
'from': 'poster <poster@example.invalid>',
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
'message-id': `<${id()}>`,
':bytes': '700000',
},
GROUP,
n
)!
)
}
async function dropSignature(tool: string, group = GROUP) {
await db.table('upload_signatures').insert({ tool, group_name: group, probed: 500, named: 0, dropped: true })
GarbageFilter.reset()
}
/** A random-named release with one stored file, as seeding leaves it. */
async function storedRelease(release: Record<string, unknown> = {}, file: Record<string, unknown> = {}) {
const now = new Date()
n++
const [r] = await db
.table('releases')
.insert({ release_key: `garbage${n}`.padEnd(40, 'g'), name: 'Hs7dKq2LmZ9xWv3RtYp1Qa', source: 'stem', groups: JSON.stringify([GROUP]), first_seen_at: now, last_seen_at: now, ...release })
.returning(['id', 'release_key'])
const [{ id }] = await FileStore.insert([
{ filename: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', poster: 'p@ngPost.com', date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: JSON.stringify([GROUP]),
segments: [{ number: 1, id: `${n.toString(16).padStart(32, 'a')}@ngPost`, size: 10 }], created_at: now, updated_at: now, ...file },
])
return { releaseId: Number(r.id), releaseKey: r.release_key as string, fileId: id }
}
test.group('Garbage filter (D42)', (group) => {
const explore = indexerConfig.garbage.explorePercent
group.each.setup(() => {
// Nothing explored, so a test's release is dropped whatever its key hashes to.
indexerConfig.garbage.explorePercent = 0
GarbageFilter.reset()
return testUtils.db().withGlobalTransaction()
})
group.each.teardown(() => {
indexerConfig.garbage.explorePercent = explore
GarbageFilter.reset()
})
group.teardown(() => QueueService.closeAll())
test('posting tools are told apart by their Message-IDs', ({ assert }) => {
assert.equal(uploadTool('FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'), 'nyuu')
assert.equal(uploadTool('b94268bc702040f7b811d5318f727e4f@ngPost'), 'ngPost')
assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
assert.equal(uploadTool('seed-1-12@test'), 'other')
assert.equal(uploadTool(null), 'other')
})
test('the explored share is stable per key and about the percentage', ({ assert }) => {
assert.equal(explored('some-release', 2), explored('some-release', 2))
assert.isFalse(explored('some-release', 0))
const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
assert.isAbove(kept, 150)
assert.isBelow(kept, 250)
})
test('seeding skips random-named files of a dropped signature, not readable or other ones', async ({ assert }) => {
await dropSignature('ngPost')
const aggregator = new FileAggregator<IngestSegment>()
const files = aggregator
.add([...upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2), ...upload('Some.Show.S01E01.1080p.WEB.part01.rar', 2), ...upload('Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar', 2, 'nyuu')])
.map(toSeedFile)
assert.lengthOf(files, 3)
const result = await SeedService.store(files)
assert.include(result, { files: 2, dropped: 1 })
assert.isNull(await db.from('files').where('filename', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
assert.isNull(await db.from('releases').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa').first())
assert.isNotNull(await db.from('files').where('filename', 'Some.Show.S01E01.1080p.WEB.part01.rar').first())
assert.isNotNull(await db.from('files').where('filename', 'Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar').first())
})
test('ingest skips them too, and a release with a readable file is kept whole', async ({ assert }) => {
await dropSignature('ngPost')
await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2))
assert.isNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
// Off: everything is stored.
indexerConfig.garbage.enabled = false
try {
await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar', 2))
} finally {
indexerConfig.garbage.enabled = true
}
assert.isNotNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar').first())
})
test('a release dropped after its probe is not stored again', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const files = aggregator.add(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)).map(toSeedFile)
await db.table('dropped_releases').insert({ release_key: files[0].release!.key, reason: 'encrypted' })
assert.include(await SeedService.store(files), { files: 0, dropped: 1 })
})
test('after its probe, an encrypted random-named release without name or password is deleted', async ({ assert }) => {
const garbage = await storedRelease({ encrypted: true })
assert.equal(await GarbageFilter.afterProbe(garbage.fileId, 'rar5'), 'encrypted')
assert.isNull(await db.from('files').where('id', garbage.fileId).first())
assert.isNull(await db.from('releases').where('id', garbage.releaseId).first())
assert.equal((await db.from('dropped_releases').where('release_key', garbage.releaseKey).first()).reason, 'encrypted')
const container = await storedRelease({ container: 'rclone', encrypted: true })
assert.equal(await GarbageFilter.afterProbe(container.fileId, 'rclone'), 'rclone')
const missing = await storedRelease()
assert.equal(await GarbageFilter.afterProbe(missing.fileId, 'missing'), 'missing')
})
test('kept after the probe: a password, a real name, other probes waiting, nothing unreadable found', async ({ assert }) => {
const password = await storedRelease({ encrypted: true, password: 'secret', password_source: 'spot' })
assert.isNull(await GarbageFilter.afterProbe(password.fileId, 'rar5'))
const named = await storedRelease({ encrypted: true, name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
assert.isNull(await GarbageFilter.afterProbe(named.fileId, 'rar5'))
const waiting = await storedRelease({ encrypted: true })
const now = new Date()
await db.table('binaries').insert({ file_key: `waiting${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.par2', total_parts: 1, groups: '[]',
first_seen_at: now, last_seen_at: now, release_id: waiting.releaseId, awaiting_inspection: true })
assert.isNull(await GarbageFilter.afterProbe(waiting.fileId, 'rar5'))
const readable = await storedRelease()
assert.isNull(await GarbageFilter.afterProbe(readable.fileId, 'rar5'))
assert.isNotNull(await db.from('releases').where('id', readable.releaseId).first())
})
test('learns dropped signatures from probed releases, and keeps a verdict without enough probes', async ({ assert }) => {
const { minProbes } = indexerConfig.garbage
const now = new Date()
const releases = await db
.table('releases')
.insert(Array.from({ length: minProbes }, (_, i) => ({ release_key: `learn${i}`.padEnd(40, 'l'), name: `Hs7dKq2LmZ9xWv3RtYp${i}`, groups: '[]', first_seen_at: now, last_seen_at: now })))
.returning('id')
await db.table('signature_probes').insert(releases.map((r) => ({ release_id: r.id, tool: 'ngPost', group_name: GROUP })))
// Five probes of another signature: too few to decide.
await db.table('signature_probes').insert(releases.slice(0, 5).map((r, i) => ({ release_id: Number(r.id) + 1_000_000_000 + i, tool: 'nyuu', group_name: GROUP })))
const learned = await GarbageFilter.learn()
assert.include(learned.changed.join('\n'), `ngPost ${GROUP}: dropped`)
const rows = await db.from('upload_signatures').where('group_name', GROUP).orderBy('tool')
assert.deepEqual(rows.map((r) => [r.tool, r.dropped, r.probed]), [['ngPost', true, minProbes], ['nyuu', false, 5]])
// One real name in a hundred is above the 0.5% limit: kept again.
await db.from('releases').where('id', releases[0].id).update({ name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
await GarbageFilter.learn()
assert.isFalse((await db.from('upload_signatures').where({ tool: 'ngPost', group_name: GROUP }).first()).dropped)
})
test('a stored file of a dropped signature is not probed, and stops waiting for it', async ({ assert }) => {
const stored = await storedRelease()
const now = new Date()
await db.table('binaries').insert({ file_key: `skip${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', total_parts: 1, groups: '[]',
first_seen_at: now, last_seen_at: now, release_id: stored.releaseId, file_id: stored.fileId, awaiting_inspection: true })
assert.isNotNull(await ContentService.prepare(stored.fileId))
await dropSignature('ngPost')
assert.isNull(await ContentService.prepare(stored.fileId))
assert.isNull((await db.from('binaries').where('file_id', stored.fileId).first()).awaiting_inspection)
})
})
tests/functional/ingest.spec.ts
-168
View File
@@ -1,168 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import QueueService from '#services/QueueService'
import { fileKey } from '#lib/subject/index'
import FileStore from '#services/FileStore'
function segment(overrides: Partial<IngestSegment>): IngestSegment {
const poster = 'ingest-test@example.invalid'
const subject = '"test-file.bin" yEnc'
return {
fileKey: fileKey(poster, subject, 3),
poster,
subject,
name: 'test-file.bin',
totalParts: 3,
number: 1,
messageId: 'part1@test',
bytes: 100,
groups: ['alt.binaries.test'],
...overrides,
}
}
async function binaryFor(key: string) {
return db.from('binaries').where('file_key', key).first()
}
// None of these cases complete a binary, so nothing is queued in Redis.
test.group('IngestService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
// IngestService opens the BullMQ queues; close them so the test process can exit.
group.teardown(() => QueueService.closeAll())
test('existing binaries are only rewritten when something changes', async ({ assert }) => {
const first = segment({ number: 1, postedAt: new Date('2026-09-28T12:00:00Z') })
await IngestService.ingest([first])
const before = await binaryFor(first.fileKey)
// A later part within TOUCH_MS: nothing to change, so the row is left alone.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', postedAt: new Date('2026-09-28T12:05:00Z') })])
const same = await binaryFor(first.fileKey)
assert.equal(+same.last_seen_at, +before.last_seen_at)
assert.equal(await db.from('parts').where('binary_id', before.id).count('* as n').then((r) => Number(r[0].n)), 2)
// An earlier post date and a stale last_seen_at are merged.
await db.from('binaries').where('id', before.id).update({ last_seen_at: new Date(Date.now() - 60 * 60 * 1000) })
await IngestService.ingest([segment({ number: 1, postedAt: new Date('2026-09-28T11:00:00Z') })])
const after = await binaryFor(first.fileKey)
assert.equal(after.posted_at.toISOString(), '2026-09-28T11:00:00.000Z')
assert.isAbove(+after.last_seen_at, Date.now() - 60_000)
})
test('a binary that is a file already takes no new header parts', async ({ assert }) => {
await IngestService.ingest([segment({ number: 1 })])
const binary = await binaryFor(segment({}).fileKey)
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'test-file.bin', poster: 'x', date: now.getTime(), parts: 3, size: 300, groups: '[]', segments: [], created_at: now, updated_at: now },
])
await db.from('binaries').where('id', binary.id).update({ file_id: file.id })
await db.from('parts').where('binary_id', binary.id).delete()
// The same articles again (a crosspost, a backfill range): nothing to store.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test' }), segment({ number: 3, messageId: 'part3@test' })])
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 0)
// Body-derived parts (with a per-part CRC) are still kept.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', pcrc32: 'abcd1234' })])
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 1)
})
test('ingesting the same segments twice is a no-op', async ({ assert }) => {
const segments = [segment({ number: 1 }), segment({ number: 2, messageId: 'part2@test' })]
await IngestService.ingest(segments)
await IngestService.ingest(segments)
const binary = await binaryFor(segments[0].fileKey)
const [{ count }] = await db.from('parts').where('binary_id', binary.id).count('* as count')
assert.equal(Number(count), 2)
assert.isNull(binary.completed_at)
assert.isNull(binary.conflicted_at)
})
test('a weak identity with two articles for one part number is flagged and never completes', async ({
assert,
}) => {
const weak = { weakIdentity: true }
await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'volume-a-1@test' })])
// Same name, poster and size, but another file: the second volume of a set.
await IngestService.ingest([
segment({ ...weak, number: 1, messageId: 'volume-b-1@test' }),
segment({ ...weak, number: 2, messageId: 'volume-b-2@test' }),
segment({ ...weak, number: 3, messageId: 'volume-b-3@test' }),
])
const binary = await binaryFor(segment({}).fileKey)
assert.isNotNull(binary.conflicted_at)
assert.isNull(binary.completed_at, 'all part numbers are present, but it must not complete')
})
test('a conflict inside a single batch is detected too', async ({ assert }) => {
await IngestService.ingest([
segment({ weakIdentity: true, number: 2, messageId: 'a@test' }),
segment({ weakIdentity: true, number: 2, messageId: 'b@test' }),
])
assert.isNotNull((await binaryFor(segment({}).fileKey)).conflicted_at)
})
test('on the subject path a second article for a part is a repost, not a conflict', async ({
assert,
}) => {
await IngestService.ingest([segment({ number: 1, messageId: 'original@test' })])
await IngestService.ingest([segment({ number: 1, messageId: 'repost@test' })])
const binary = await binaryFor(segment({}).fileKey)
assert.isNull(binary.conflicted_at)
const part = await db.from('parts').where({ binary_id: binary.id, number: 1 }).first()
assert.equal(part.message_id, 'original@test')
})
test('an obfuscated file joins a known par2 set of exactly its size', async ({ assert }) => {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'k'.repeat(40), name: 'Some.Show.S01E01', source: 'par2', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
await db.table('release_files').insert({ release_id: release.id, par2_file_id: 'f'.repeat(32), name: 'Some.Show.S01E01.mkv', size: 5467240532, md5_16k: '0'.repeat(32) })
const weak = { weakIdentity: true, postedAt: now, release: null }
await IngestService.ingest([
segment({ ...weak, fileKey: 'a'.repeat(40), size: 5467240532, totalParts: 7628, messageId: 'match@test' }),
segment({ ...weak, fileKey: 'b'.repeat(40), size: 5467240533, totalParts: 7628, messageId: 'other@test' }),
])
const matched = await binaryFor('a'.repeat(40))
const other = await binaryFor('b'.repeat(40))
assert.equal(Number(matched.release_id), Number(release.id))
assert.isNull(other.release_id)
})
test('an identical repost (same part CRC) is not a conflict', async ({ assert }) => {
const weak = { weakIdentity: true, pcrc32: 'abcd1234', partSize: 700 }
await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'first@test' })])
await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'repost@test' })])
const binary = await binaryFor(segment({}).fileKey)
assert.isNull(binary.conflicted_at)
await IngestService.ingest([segment({ ...weak, pcrc32: 'ffff0000', number: 1, messageId: 'other@test' })])
assert.isNotNull((await binaryFor(segment({}).fileKey)).conflicted_at)
})
test('a release takes its file total from whichever file carries the counter', async ({ assert }) => {
const { releaseIdentity } = await import('#lib/release/index')
const poster = 'archiver@example.invalid'
const file = (subject: string, name: string, key: string) =>
segment({ fileKey: key.padEnd(40, 'k'), poster, subject, name, totalParts: 1, messageId: `${key}@test`,
release: releaseIdentity({ poster, subject, name, weak: false }) })
// The counter-less par2 index comes first in the batch.
await IngestService.ingest([
file('"x.tar.zst.par2" yEnc 3100', 'x.tar.zst.par2', 'index'),
file('[1/2] - "x.tar.zst" yEnc 1000', 'x.tar.zst', 'archive'),
])
const release = await db.from('releases').where('name', 'x').where('poster', poster).first()
assert.equal(release.file_total, 2)
})
})
tests/functional/inspection_queue.spec.ts
-36
View File
@@ -1,36 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import QueueService from '#services/QueueService'
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
test.group('Inspection queue', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
test('the backlog stays in Postgres; BullMQ gets what fits, in priority order', async ({ assert }) => {
const queue = QueueService.collectionQueue
await queue.drain()
await InspectionQueue.add([9_000_001, 9_000_002], INSPECTION_PRIORITY.low)
await InspectionQueue.add([9_000_003])
await InspectionQueue.add([9_000_004], INSPECTION_PRIORITY.targeted)
// Queued again with a higher priority: one row, the higher priority.
await InspectionQueue.add([9_000_001], INSPECTION_PRIORITY.targeted)
assert.equal(await InspectionQueue.size(), 4)
const max = InspectionQueue.feedMax
;(InspectionQueue as any).feedMax = 2
try {
assert.equal(await InspectionQueue.feed(), 2)
const waiting = (await queue.getWaiting()).map((job) => job.data.fileId).sort()
assert.deepEqual(waiting, [9_000_001, 9_000_004])
// Full: nothing more until the workers take some.
assert.equal(await InspectionQueue.feed(), 0)
const left = (await db.from('inspection_queue').select('file_id')).map((r) => Number(r.file_id)).sort()
assert.deepEqual(left, [9_000_002, 9_000_003])
} finally {
;(InspectionQueue as any).feedMax = max
await queue.drain()
}
})
})
tests/functional/names.spec.ts
-94
View File
@@ -1,94 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import testUtils from '@adonisjs/core/services/test_utils'
import NameService from '#services/NameService'
async function release(name: string, files: { name: string; size: number; md5_16k: string }[]) {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: `${name}`.padEnd(40, 'x').slice(0, 40), name, source: 'par2', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
for (const [i, f] of files.entries()) {
await db.table('release_files').insert({ release_id: row.id, par2_file_id: String(i).padStart(32, '0'), name: f.name, size: f.size, md5_16k: f.md5_16k })
}
return Number(row.id)
}
async function binary(key: string, fields: Record<string, unknown>) {
const now = new Date()
const [row] = await db
.table('binaries')
.insert({ file_key: key.padEnd(40, '0'), poster: 'x', subject: '"x" yEnc', name: 'Xq8r2KdLm3Zw91PqA7', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, ...fields })
.returning('id')
return Number(row.id)
}
test.group('NameService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('an exact match leaves a random-named single-file release for the par2 set', async ({ assert }) => {
const season = await release('SEAL.Team.S03', [
{ name: 'SEAL.Team.S03E18.1080p.WEB-DL-nitrato.mkv', size: 939_684_767, md5_16k: 'd'.repeat(32) },
])
const now = new Date()
const [weak] = await db
.table('releases')
.insert({ release_key: 'w'.repeat(40), name: 'mOzyyxCyYUjAbCdEf12', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [file] = await FileStore.insert([
{ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], release_id: weak.id, created_at: now, updated_at: now },
])
const episode = await binary('ep18', { size: 939_684_767, md5_16k: 'd'.repeat(32), release_id: weak.id, file_id: file.id })
await NameService.resolve([episode])
const moved = await db.from('binaries').where('id', episode).first()
assert.equal(Number(moved.release_id), season)
assert.equal(moved.real_name, 'SEAL.Team.S03E18.1080p.WEB-DL-nitrato.mkv')
assert.equal(Number((await db.from('files').where('id', file.id).first()).release_id), season)
})
test('a file in a real grouping keeps it', async ({ assert }) => {
await release('Other.Set', [{ name: 'Other.Show.S01E01.mkv', size: 5_000_000, md5_16k: 'e'.repeat(32) }])
const now = new Date()
const [nzb] = await db
.table('releases')
.insert({ release_key: 'n'.repeat(40), name: 'Qz8r2KdLm3Zw91PqA7x', source: 'nzb', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const id = await binary('nzbfile', { size: 5_000_000, md5_16k: 'e'.repeat(32), release_id: nzb.id })
await NameService.resolve([id])
assert.equal(Number((await db.from('binaries').where('id', id).first()).release_id), Number(nzb.id))
})
test('names equal-sized volumes exactly by their first-16k fingerprint', async ({ assert }) => {
const releaseId = await release('Some.Movie', [
{ name: 'Some.Movie.part1.rar', size: 50_000_000, md5_16k: 'a'.repeat(32) },
{ name: 'Some.Movie.part2.rar', size: 50_000_000, md5_16k: 'b'.repeat(32) },
])
const one = await binary('one', { size: 50_000_000, md5_16k: 'b'.repeat(32) })
const two = await binary('two', { size: 50_000_000, md5_16k: 'a'.repeat(32) })
const stranger = await binary('three', { size: 50_000_000, md5_16k: 'c'.repeat(32) })
await NameService.resolve([one, two, stranger])
const rows = await db.from('binaries').whereIn('id', [one, two, stranger]).orderBy('id')
assert.deepEqual(rows.map((r) => [r.real_name, r.release_id && Number(r.release_id)]), [
['Some.Movie.part2.rar', releaseId],
['Some.Movie.part1.rar', releaseId],
[null, null],
])
})
test('without a fingerprint, names by size only when the size is unique in the release', async ({ assert }) => {
const releaseId = await release('Other', [
{ name: 'Other.mkv', size: 700_000_000, md5_16k: 'd'.repeat(32) },
{ name: 'Other.part1.rar', size: 10_000_000, md5_16k: 'e'.repeat(32) },
{ name: 'Other.part2.rar', size: 10_000_000, md5_16k: 'f'.repeat(32) },
])
const video = await binary('video', { size: 700_000_000, release_id: releaseId })
const volume = await binary('volume', { size: 10_000_000, release_id: releaseId })
await NameService.resolve([video, volume])
assert.equal((await db.from('binaries').where('id', video).first()).real_name, 'Other.mkv')
assert.isNull((await db.from('binaries').where('id', volume).first()).real_name)
})
})
tests/functional/newznab.spec.ts
-91
View File
@@ -1,91 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import env from '#start/env'
import testUtils from '@adonisjs/core/services/test_utils'
let counter = 0
async function seed(name: string, opts: { fileTotal?: number; files?: number } = {}) {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: `nz${++counter}`.padEnd(40, 'n'), name, source: 'stem', poster: 'poster@example.invalid', groups: '["alt.binaries.test"]', file_total: opts.fileTotal ?? null, posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
for (let i = 0; i < (opts.files ?? 1); i++) {
await FileStore.insert([{
filename: `${name}.part${i + 1}.rar`, poster: 'poster@example.invalid', date: now.getTime(), parts: 1, size: 1_000_000,
release_id: release.id, groups: '["alt.binaries.test"]', segments: [{ number: 1, id: `${name}-${i}@test`, size: 1_000_000 }],
created_at: now, updated_at: now,
}])
}
return Number(release.id)
}
const ids = (xml: string) => [...xml.matchAll(/<guid isPermaLink="false">(\d+)<\/guid>/g)].map((m) => Number(m[1]))
test.group('Newznab API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('caps lists search functions and categories', async ({ client, assert }) => {
const response = await client.get('/api').qs({ t: 'caps' })
response.assertStatus(200)
assert.include(response.text(), '<tv-search available="yes" supportedParams="q,season,ep"/>')
assert.include(response.text(), '<category id="5000" name="TV">')
assert.include(response.text(), '<subcat id="5040" name="HD"/>')
})
test('tvsearch matches show, season and episode, with Newznab attributes', async ({ client, assert }) => {
const e02 = await seed('Nzbtest.Show.S01E02.1080p.WEB-DL-GRP')
const e03 = await seed('Nzbtest.Show.S01E03.1080p.WEB-DL-GRP')
const movie = await seed('Nzbtest.Show.The.Movie.2020.1080p.BluRay.x264-GRP')
const response = await client.get('/api').qs({ t: 'tvsearch', q: 'nzbtest show', season: '1', ep: '2' })
response.assertStatus(200)
const body = response.text()
assert.deepEqual(ids(body), [e02])
assert.include(body, '<newznab:attr name="category" value="5000"/>')
assert.include(body, '<newznab:attr name="category" value="5040"/>')
assert.include(body, '<newznab:attr name="season" value="S01"/>')
assert.include(body, `/api?t=get&amp;id=${e02}`)
assert.include(body, 'type="application/x-nzb"')
const season = await client.get('/api').qs({ t: 'tvsearch', q: 'nzbtest show', season: '1' })
assert.sameMembers(ids(season.text()), [e02, e03])
const movies = await client.get('/api').qs({ t: 'movie', q: 'nzbtest show' })
assert.deepEqual(ids(movies.text()), [movie])
})
test('leaves out obfuscated names and releases known to be incomplete', async ({ client, assert }) => {
const readable = await seed('Nzbtest.Other.2021.720p.WEB-DL-GRP')
const incomplete = await seed('Nzbtest.Partial.2021.720p.WEB-DL-GRP', { fileTotal: 5, files: 2 })
const random = await seed('Xq8r2KdLm3Zw91PqA7nzbtest')
const found = ids((await client.get('/api').qs({ t: 'search', q: 'nzbtest' })).text())
assert.include(found, readable)
assert.notInclude(found, incomplete)
assert.notInclude(found, random)
})
test('get returns the NZB, errors use Newznab codes', async ({ client, assert }) => {
const id = await seed('Nzbtest.Get.S02E01.720p.HDTV-GRP')
const nzb = await client.get('/api').qs({ t: 'get', id: String(id) })
nzb.assertStatus(200)
assert.include(nzb.header('content-type'), 'application/x-nzb')
assert.include(nzb.text(), 'Nzbtest.Get.S02E01.720p.HDTV-GRP-0@test')
assert.include((await client.get('/api').qs({ t: 'get', id: '999999999' })).text(), 'code="300"')
assert.include((await client.get('/api').qs({ t: 'nonsense' })).text(), 'code="202"')
})
test('requires the API key when one is configured', async ({ client, assert }) => {
const previous = env.get('NEWZNAB_API_KEY')
env.set('NEWZNAB_API_KEY', 'test-key' as any)
try {
assert.include((await client.get('/api').qs({ t: 'caps' })).text(), 'code="100"')
assert.include((await client.get('/api').qs({ t: 'caps', apikey: 'test-key' })).text(), '<caps>')
const id = await seed('Nzbtest.Key.S01E01.720p.HDTV-GRP')
const search = await client.get('/api').qs({ t: 'search', q: 'nzbtest key', apikey: 'test-key' })
assert.include(search.text(), `id=${id}&amp;apikey=test-key`)
} finally {
env.set('NEWZNAB_API_KEY', previous as any)
}
})
})
tests/functional/nzb_import.spec.ts
-137
View File
@@ -1,137 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import type { Segment } from '#lib/nzb/index'
import testUtils from '@adonisjs/core/services/test_utils'
import NzbImportService, { nzbFileName } from '#services/NzbImportService'
import SearchService from '#services/SearchService'
const XML = `<?xml version="1.0" encoding="utf-8"?>
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
<head><meta type="name">Real.Name.2026.1080p.WEB-DL-GRP</meta><meta type="password">hunter2</meta></head>
<file poster="p@x" date="1790598923" subject="Wond40 &#x22;Nachbe&#x22; Xq8r2KdLm3Zw91PqA7.part1.rar&#x22; yEnc (1/2)">
<groups><group>alt.binaries.bloaf</group></groups>
<segments><segment bytes="700" number="1">known-1@x</segment><segment bytes="700" number="2">known-2@x</segment></segments>
</file>
<file poster="p@x" date="1790598923" subject="&#x22;Xq8r2KdLm3Zw91PqA7.part2.rar&#x22; yEnc (1/1)">
<groups><group>alt.binaries.cores</group></groups>
<segments><segment bytes="500" number="1">new-1@x</segment></segments>
</file>
</nzb>`
async function file(fields: Record<string, unknown> & { segments: Segment[] }) {
const now = new Date()
const [{ id }] = await FileStore.insert([{ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }])
return db.from('files').where('id', id).first()
}
test.group('NzbImportService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('names entries despite odd quoting', ({ assert }) => {
assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar')
assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2')
// Spot NZBs without quotes.
assert.equal(nzbFileName('Adventure.part0055.rar - [056/175] (1/1)'), 'Adventure.part0055.rar')
assert.equal(nzbFileName('LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2 (1/0) (1/0)'), 'LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2')
assert.equal(nzbFileName('Reacher.S04.vol001+02.par2 yEnc'), 'Reacher.S04.vol001+02.par2')
assert.equal(nzbFileName('06 - A New Land, a New Music.flac Mehmet Ali Sanlikol - The Rise Up (2020) (01/35)'), '06 - A New Land, a New Music.flac')
assert.equal(nzbFileName('[01/10] - Some.Release.2026.part01.rar yEnc (1/50) 35000000'), 'Some.Release.2026.part01.rar')
assert.equal(nzbFileName('Mozart Zauberfloete 2026 mp4 720p.zip.001'), 'Mozart Zauberfloete 2026 mp4 720p.zip.001')
assert.equal(nzbFileName('LibreOffice_7.6.4_Win_x86-64.msi.vol0000'), 'LibreOffice_7.6.4_Win_x86-64.msi.vol0000')
})
test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', segments: [{ number: 1, id: 'nzb-1@x', size: 2000 }] })
const result = (await NzbImportService.import(posted, XML))!
assert.equal(result.releaseId, Number(release.id))
assert.include(result, { listed: 2, known: 1, added: 1, password: true })
const updated = await db.from('releases').where('id', release.id).first()
assert.equal(updated.name, 'Real.Name.2026.1080p.WEB-DL-GRP')
assert.equal(updated.password, 'hunter2')
assert.equal(updated.file_total, 2)
const files = await db.from('files').where('release_id', release.id).orderBy('filename').select('filename', 'groups')
assert.deepEqual(files.map((f) => f.filename), ['Xq8r2KdLm3Zw91PqA7.nzb', 'Xq8r2KdLm3Zw91PqA7.part1.rar', 'Xq8r2KdLm3Zw91PqA7.part2.rar'])
// Our own NZB passes the password on.
const nzb = await SearchService.nzb(Number(release.id))
assert.include(nzb!.xml, '<meta type="password">hunter2</meta>')
// Importing again changes nothing.
const again = (await NzbImportService.import(posted, XML))!
assert.equal(again.added, 0)
})
test('NZBs from one archive become separate releases named after their entries', async ({ assert }) => {
const nzbXml = (id: string) => `<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
<file poster="p@x" date="1790598923" subject="&#x22;${id}.flac&#x22; yEnc (1/1)">
<groups><group>alt.binaries.x</group></groups>
<segments><segment bytes="500" number="1">${id}@x</segment></segments>
</file></nzb>`
const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', segments: [{ number: 1, id: 'rar-1@x', size: 2000 }] })
const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))!
const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))!
assert.notEqual(a.releaseId, b.releaseId)
assert.equal(a.releaseName, 'The Kinks - Misfits-LP-1978')
assert.equal(b.releaseName, 'The Kinks - Sleepwalker-LP-1977')
const container = await db.from('files').where('id', archive.id).first()
assert.isNull(container.release_id)
})
test('a posted NZB alone in its own release moves to the release it describes', async ({ assert }) => {
const now = new Date()
const insert = async (key: string, name: string) =>
(await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id
const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT')
const lonely = await insert('lonely', '.nzb')
await file({ filename: 'x.part1.rar', release_id: target, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
const posted = await file({ filename: '.nzb', release_id: lonely, segments: [{ number: 1, id: 'nzb-2@x', size: 2000 }] })
await NzbImportService.import(posted, XML)
assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target))
assert.notExists(await db.from('releases').where('id', lonely).first())
})
test('a trusted NZB (a spot) takes its files into its own release, named after it', async ({ assert }) => {
const now = new Date()
const insertRelease = async (key: string, name: string, source: string) =>
(
await db
.table('releases')
.insert({ release_key: key.repeat(40).slice(0, 40), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
)[0].id
// Name rules had put the spot's first volume into a season release with another episode.
const season = await insertRelease('s', 'zxcvvbgnhgjytygf', 'stem')
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: season, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
await file({ filename: 'other.episode.rar', release_id: season, segments: [{ number: 1, id: 'other-1@x', size: 700 }] })
const spotFile = { id: null, filename: 'x.nzb', poster: 'Bart', groups: [], date: now.getTime(), first_message_id: 'spot1@spot.net' }
const result = (await NzbImportService.import(spotFile, XML, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', moveContainer: false, trusted: true }))!
assert.notEqual(result.releaseId, Number(season))
const own = await db.from('releases').where('id', result.releaseId).first()
assert.include(own, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', file_count: 2 })
const left = await db.from('files').where('release_id', season).select('filename')
assert.deepEqual(left.map((f) => f.filename), ['other.episode.rar'])
// A par2 or archive name no longer replaces a spot's title.
const { isBetterName } = await import('#services/NameService')
assert.isTrue(isBetterName('zxcvvbgnhgjytygf', 'Better.Name.S03E285'))
// Another spot for the same files (a repost) joins the first one's release instead of taking them.
const repost = { ...spotFile, first_message_id: 'spot2@spot.net' }
const second = (await NzbImportService.import(repost, XML, { name: 'Repost', source: 'spot', moveContainer: false, trusted: true }))!
assert.equal(second.releaseId, result.releaseId)
assert.equal(second.releaseName, 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264')
})
})
tests/functional/par2_attach.spec.ts
-56
View File
@@ -1,56 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import Par2Service from '#services/Par2Service'
import type { Par2Set } from '#lib/par2/index'
const VOLUME = 10_000_952 // a standard split-volume size that recurs across posts
async function release(key: string, name: string) {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), name, source: 'weak-stem', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
return Number(row.id)
}
async function binary(key: string, fields: Record<string, unknown>) {
const now = new Date()
const [row] = await db
.table('binaries')
.insert({ file_key: key.padEnd(40, '0'), poster: 'x', subject: '"x" yEnc', name: 'Xq8r2KdLm3Zw91PqA7', total_parts: 14, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now, ...fields })
.returning(['id', 'poster', 'groups', 'posted_at'])
return row
}
function set(id: string, files: { name: string; md5_16k: string }[]): Par2Set {
return {
recoverySetId: id.padEnd(32, '0'),
files: files.map((f, i) => ({ fileId: `${id}${i}`.padEnd(32, '0'), md5: '0'.repeat(32), md5_16k: f.md5_16k, length: VOLUME, name: f.name })),
}
}
test.group('Par2Service.attach', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a set does not join another post just because a size matches', async ({ assert }) => {
const other = await release('other', 'OtherPost')
await binary('other-vol', { size: VOLUME, md5_16k: 'a'.repeat(32), release_id: other })
const par2 = await binary('par2-b', { size: 50_000, total_parts: 1 })
const result = await Par2Service.attach(par2, set('b', [{ name: 'PostB.part1.rar', md5_16k: 'b'.repeat(32) }]))
assert.notEqual(result.releaseId, other)
assert.equal(await db.from('release_files').where('release_id', other).count('* as n').first().then((r) => Number(r.n)), 0)
})
test('an exactly matching file pulls the set into its release and gets its real name', async ({ assert }) => {
const stemRelease = await release('stem', 'Xq8r2KdLm3Zw91PqA7')
const vol = await binary('vol', { size: VOLUME, md5_16k: 'c'.repeat(32), release_id: stemRelease })
const par2 = await binary('par2-c', { size: 50_000, total_parts: 1 })
const result = await Par2Service.attach(par2, set('c', [{ name: 'Real.Name.part1.rar', md5_16k: 'c'.repeat(32) }]))
assert.equal(result.releaseId, stemRelease)
assert.equal((await db.from('binaries').where('id', vol.id).first()).real_name, 'Real.Name.part1.rar')
})
})
tests/functional/random_names.spec.ts
-27
View File
@@ -1,27 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import { looksRandom } from '#services/ArchiveService'
// Names seen on homelab (LOG.md 2026-10-01 (3)).
const RANDOM = [
'5d4063b423f57e1eadda1068b0091409', 'mOzyyxCyYUjAbCdEf12', '6327d9fa2d6acf3e9a61fb053359d5e7.1790574045',
'e63afdb4-6a1f-417d-aa1b-b14c0ea1b81d', '1650777167', '83111763 n', 'prvjr7hapd0i1ny14pz4', '8huowjcdamwbetufib9f',
'071vl0l120ndtjkjkuh34a2g6aqsak750f55dkudeaea8s4bivjm1l32fd3uduk0dab2t82hp553i29nfjvjt3redp', 'wHJ7s71sAGE', 'EmFqTfL8rxV',
'VIDEO_TS', 'Sample', 'CD1', 'Disc 2', 'Subs',
]
const READABLE = [
'Our.America.with.Lisa.Ling.2011.S01E01.Faith.Healers.1080p.DSCP.WEB-DL.DDP2.0.H.264-FUZEER', '00-graham_barham-club_country',
'1790274935_nicovideo_jp_watch_sm26602185', 'StarWars1977', 'nyuu_live_roundtrip', 'Makaja_Gonzales-Freestyle-WEB-2021-XTC_iNT',
'The Tom and Jerry Show (2014) S02E13 - Slap Happy Birthday', 'Bondgenoten', 'S01E02', '2026', 'Samples.Of.Love.2020.1080p', 'Subs.and.Dubs',
]
test.group('random names', () => {
test('looksRandom and SQL is_readable_name agree', async ({ assert }) => {
for (const name of [...RANDOM, ...READABLE]) {
const { rows } = await db.rawQuery('SELECT is_readable_name(?) AS readable', [name])
const random = RANDOM.includes(name)
assert.equal(looksRandom(name), random, `looksRandom(${name})`)
assert.equal(rows[0].readable, !random, `is_readable_name(${name})`)
}
})
})
tests/functional/release_complete.spec.ts
-94
View File
@@ -1,94 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
/** A release with `n` files (each with a binary of size 100000 + i), named f<i>.bin. */
async function release(n: number) {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: `complete-${n}`.padEnd(40, '0'), name: `Complete.Test.${n}`, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const releaseId = Number(row.id)
await db.rawQuery(
`WITH f AS (
INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
SELECT 'f' || i || '.bin', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'complete-' || ?::int || '-' || i, now(), now()
FROM generate_series(1, ?::int) i
RETURNING id, filename)
INSERT INTO binaries (file_key, poster, subject, name, total_parts, groups, first_seen_at, last_seen_at, size, file_id, release_id)
SELECT 'complete-b-' || ?::int || '-' || f.id, 'x', 'x', f.filename, 1, '[]', now(), now(), 100000 + substr(f.filename, 2, length(f.filename) - 5)::int, f.id, ?::bigint
FROM f`,
[releaseId, n, n, n, releaseId]
)
return releaseId
}
/** The par2 listing of `count` files f1..f<count> (sizes as in release()), in one INSERT, as Par2Service.attach does. */
async function listing(releaseId: number, count: number, offset = 0) {
await db.rawQuery(
`INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
SELECT ?::bigint, md5(i::text), 'f' || i || '.bin', 100000 + i, md5(i::text), 'set' FROM generate_series(1 + ?::int, ?::int + ?::int) i`,
[releaseId, offset, count, offset]
)
// Triggers only note the release; gc:scheduler's sweep recomputes it.
await ReleaseCompleteness.sweep()
return Boolean((await db.from('releases').where('id', releaseId).first()).complete)
}
test.group('release completeness', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('file inserts keep counts, size, extensions and search text current, once per statement', async ({ assert }) => {
const id = await release(3)
const row = await db.from('releases').where('id', id).first()
assert.equal(Number(row.file_count), 3)
assert.deepEqual(row.extensions, ['bin'])
// No par2 listing, no file total, not complete: its completeness can't change, so it isn't noted.
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
assert.equal(Number(count), 0)
})
test('a release with a par2 listing is noted once per insert, and complete by file names alone', async ({ assert }) => {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: 'names-only'.padEnd(40, '0'), name: 'Names.Only', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const id = Number(row.id)
await db.rawQuery(
`INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
SELECT ?::bigint, md5(i::text), 'n' || i || '.rar', 100000 + i, md5(i::text), 'set' FROM generate_series(1, 3) i`,
[id]
)
await ReleaseCompleteness.sweep()
// Seeded files: no binaries, matched by name only.
await db.rawQuery(
`INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
SELECT 'n' || i || '.rar', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'names-only-' || i, now(), now() FROM generate_series(1, 3) i`,
[id]
)
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
assert.equal(Number(count), 1)
await ReleaseCompleteness.sweep()
assert.isTrue(Boolean((await db.from('releases').where('id', id).first()).complete))
})
test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
const id = await release(3)
assert.isTrue(await listing(id, 3))
const other = await release(2)
// f3 is listed but missing.
assert.isFalse(await listing(other, 3))
})
test('a par2 listing of thousands of files is checked in one go', async ({ assert }) => {
const id = await release(3000)
await ReleaseCompleteness.sweep()
const started = Date.now()
assert.isTrue(await listing(id, 3000))
// Was cubic: minutes for a few thousand rows.
assert.isBelow(Date.now() - started, 5000)
}).timeout(60_000)
})
tests/functional/release_merge.spec.ts
-42
View File
@@ -1,42 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import ReleaseMergeService from '#services/ReleaseMergeService'
let keys = 0
async function postRelease(name: string, files: string[], poster: string) {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: `merge${++keys}`.padEnd(40, 'm'), name, source: 'post', poster, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
await FileStore.insert(
files.map((f, i) => ({ filename: f, poster, date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: '[]',
segments: [{ number: 1, id: `${name}-${poster}-${i}@x`, size: 10 }], created_at: now, updated_at: now }))
)
return Number(r.id)
}
test.group('Split uploads', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('merges one upload posted under a poster per volume, not reposts or other uploads', async ({ assert }) => {
const name = 'Masters.of.the.Universe.1987.Theatrical.Cut.2160p.BDRip-GRP'
const a = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part001.rar'], 'niSSYNy5zOW2VX@VMuLu_e7Gr.bIx')
const b = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part002.rar'], 'MgrZzFLB3D@MeK4_gntYXzh9g.LBQ')
const c = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part003.rar'], 'qNGVviOMN2AJrSzeXFZqvW@VzavS9GO.Y0b')
// A repost: the same file names again.
const repostName = 'Some.Repost.2026.1080p-GRP'
const r1 = await postRelease(repostName, ['abc.part01.rar'], 'p1@x')
const r2 = await postRelease(repostName, ['abc.part01.rar'], 'p2@x')
assert.deepEqual(await ReleaseMergeService.mergeSplitPosts({ dryRun: true }), { names: 1, releases: 2 })
await ReleaseMergeService.mergeSplitPosts()
assert.equal((await db.from('releases').where('id', a).first()).file_count, 3)
assert.equal((await db.from('releases').where('id', b).first()).file_count, 0)
assert.equal((await db.from('releases').where('id', c).first()).file_count, 0)
assert.equal((await db.from('releases').where('id', r1).first()).file_count, 1)
assert.equal((await db.from('releases').where('id', r2).first()).file_count, 1)
})
})
tests/functional/search_text.spec.ts
-41
View File
@@ -1,41 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import SearchService, { searchNormalize } from '#services/SearchService'
test.group('Search text', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('normalises like SQL does', async ({ assert }) => {
assert.equal(searchNormalize(' The.Block.AU_S22E35 - 1080p '), 'the block au s22e35 1080p')
const { rows } = await db.rawQuery(`SELECT search_normalize(' The.Block.AU_S22E35 - 1080p ') AS n`)
assert.equal(rows[0].n, 'the block au s22e35 1080p')
})
test('follows files and renames, only for visible releases', async ({ assert }) => {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: 's'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7Rt', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('*')
assert.isNull(r.search_text)
// A readable file name that the release name lacks joins the search text.
await FileStore.insert([
{ filename: 'Some.Documentary.2026.1080p.WEB.part01.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
groups: '[]', segments: [{ number: 1, id: 'st1@x', size: 10 }], created_at: now, updated_at: now },
{ filename: 'Q7wErTyUiOpAsDfGhJkL9.part02.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
groups: '[]', segments: [{ number: 1, id: 'st2@x', size: 10 }], created_at: now, updated_at: now },
])
let row = await db.from('releases').where('id', r.id).first()
assert.equal(row.search_text, 'xq8r2kdlm3zw91pqa7rt some documentary 2026 1080p web')
const found = await SearchService.search({ q: 'documentary 2026.1080p' } as any)
assert.isTrue(found.releases.some((x: any) => x.id === Number(r.id)))
// A rename recomputes it.
await db.from('releases').where('id', r.id).update({ name: 'Some.Documentary.2026.1080p.WEB-GRP' })
row = await db.from('releases').where('id', r.id).first()
assert.equal(row.search_text, 'some documentary 2026 1080p web grp some documentary 2026 1080p web')
})
})
tests/functional/seed.spec.ts
-87
View File
@@ -1,87 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
import SeedService from '#services/SeedService'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile } from '#lib/seed/wire'
import { insertLimits, segmentsOf } from '#services/FileStore'
function header(subject: string, n: number) {
return {
'subject': subject,
'from': 'seeder <seed@example.invalid>',
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
'message-id': `<seed-${n}-${subject.length}@test>`,
':bytes': '700000',
}
}
test.group('SeedService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
test('stores aggregated files with their release and a completed binary', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const subjects = [1, 2].map((i) => `Seed.Test.S01E01.1080p - [1/2] - "Seed.Test.S01E01.1080p.part${i}.mkv" yEnc (%d/3)`)
const segments = subjects.flatMap((s) => [1, 2, 3].map((n) => segmentFromHeader(header(s.replace('%d', String(n)), n), 'alt.binaries.test', n)!))
const files = aggregator.add(segments).map(toSeedFile)
assert.lengthOf(files, 2)
const result = await SeedService.store(files)
assert.include(result, { files: 2, existing: 0, releases: 1 })
const stored = await db.from('files').where('filename', 'Seed.Test.S01E01.1080p.part1.mkv').first()
assert.equal(stored.parts, 3)
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
const release = await db.from('releases').where('id', stored.release_id).first()
assert.equal(release.name, segments[0].release!.name) // named as live indexing would
// A readable name isn't inspected: no binary (LOG.md 2026-10-06 (11)).
assert.isNull(await db.from('binaries').where('file_key', stored.file_key).first())
// The same files again (a crosspost, a second run): nothing new.
const again = await SeedService.store(files)
assert.include(again, { files: 0, existing: 2 })
})
test('an obfuscated file keeps a binary until inspection has fingerprinted it', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const subject = 'Xq8r2KdLm3Zw91PqA7zz [1/1] - "Xq8r2KdLm3Zw91PqA7zz.bin" yEnc (%d/2)'
const segments = [1, 2].map((n) => segmentFromHeader(header(subject.replace('%d', String(n)), n), 'alt.binaries.test', n)!)
const files = aggregator.add(segments).map(toSeedFile)
const result = await SeedService.store(files)
assert.include(result, { files: 1, inspections: 1 })
const binary = await db.from('binaries').where('file_key', files[0].fileKey).first()
assert.isTrue(binary.awaiting_inspection)
assert.isNotNull(binary.file_id)
})
test('stores a batch in several INSERTs', async ({ assert }) => {
// Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
const rows = insertLimits.rows
insertLimits.rows = 50
try {
const aggregator = new FileAggregator<IngestSegment>()
const segments = Array.from({ length: 120 }, (_, i) =>
segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
)
const files = aggregator.add(segments).map(toSeedFile)
assert.include(await SeedService.store(files), { files: 120 })
const [{ count }] = await db.from('files').whereLike('filename', 'dense.%').count('* as count')
assert.equal(Number(count), 120)
} finally {
insertLimits.rows = rows
}
})
test('fragments are kept near the range edges only', async ({ assert }) => {
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
const middle = segmentFromHeader(header(subject('middle.bin'), 2), 'alt.binaries.test', 1_500_000)!
const kept = await SeedService.storeFragments([toFragment(edge), toFragment(middle)], { start: 1_000_000, end: 2_999_999 })
assert.equal(kept, 1)
assert.isOk(await db.from('binaries').where('name', 'edge.bin').first())
assert.isNotOk(await db.from('binaries').where('name', 'middle.bin').first())
})
})
tests/functional/seed_api.spec.ts
-265
View File
@@ -1,265 +0,0 @@
import { test } from '@japa/runner'
import { gzipSync } from 'node:zlib'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import SeedOrchestrator from '#services/SeedOrchestrator'
import QueueService from '#services/QueueService'
import indexerConfig from '#config/indexer'
import SeedFetchPool from '#services/SeedFetchPool'
import { deflateSync } from 'node:zlib'
import { segmentFromHeader } from '#lib/ingest/segment'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
import type { IngestSegment } from '#lib/ingest/segment'
import FileStore from '#services/FileStore'
import RemoteContentService from '#services/RemoteContentService'
import { Job } from 'bullmq'
function header(subject: string, n: number) {
return { 'subject': subject, 'from': 'api seeder <api@example.invalid>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<api-${n}-${subject.length}@test>`, ':bytes': '500000' }
}
function batch(next: number): SeedBatch {
const segments = [1, 2].map((n) => segmentFromHeader(header(`Api.Test.2026 - "Api.Test.2026.mkv" yEnc (${n}/2)`, n), 'alt.binaries.test', n)!)
const lone = segmentFromHeader(header('Api.Test.2026 - "Api.Test.2026.nfo" yEnc (1/3)', 9), 'alt.binaries.test', 9)!
const aggregator = new FileAggregator<IngestSegment>()
return { next, headers: 3, obfuscated: 0, deferred: 0, files: aggregator.add(segments).map(toSeedFile), fragments: [toFragment(lone)] }
}
test.group('Seed API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(async () => {
await RemoteContentService.close()
await QueueService.closeAll()
})
test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('test-node')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
;(await client.post('/api/v1/seed/lease')).assertStatus(401)
const leased = await client.post('/api/v1/seed/lease').headers(auth)
leased.assertStatus(200)
const range = leased.body().range
assert.include(range, { group: 'alt.binaries.test', start: 1, end: 100 })
// As seed:node sends it: a gzip-compressed JSON body.
const posted = await fetch(`http://${process.env.HOST}:${process.env.PORT}/api/v1/seed/ranges/${range.id}/batches`, {
method: 'POST',
headers: { ...auth, 'content-type': 'application/json', 'content-encoding': 'gzip' },
body: gzipSync(JSON.stringify(batch(51))),
})
assert.equal(posted.status, 200)
assert.include(((await posted.json()) as any).stored, { files: 1 })
const file = await db.from('files').where('filename', 'Api.Test.2026.mkv').first()
assert.equal(file.parts, 2)
// The lone .nfo segment sits near the range's edge: kept as a binary (not staged) for the neighbouring range.
assert.isOk(await db.from('binaries').where('name', 'Api.Test.2026.nfo').first())
const row = await db.from('seed_ranges').where('id', range.id).first()
assert.equal(Number(row.next_id), 51)
assert.equal(Number(row.headers), 3)
;(await client.post(`/api/v1/seed/ranges/${range.id}/done`).headers(auth)).assertStatus(200)
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done')
;(await client.post('/api/v1/seed/lease').headers(auth)).assertStatus(204)
})
test('a quiet lease passes to another node, and the first one gets 409', async ({ client, assert }) => {
const first = await SeedOrchestrator.issueToken('node-a')
const second = await SeedOrchestrator.issueToken('node-b')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const range = (await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${first}` })).body().range
await db.from('seed_ranges').where('id', range.id).update({ next_id: 40, heartbeat_at: new Date(Date.now() - 60 * 60 * 1000) })
const taken = (await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${second}` })).body().range
assert.equal(taken.id, range.id)
assert.equal(taken.start, 40)
const late = await client.post(`/api/v1/seed/ranges/${range.id}/batches`).headers({ authorization: `Bearer ${first}` }).json(batch(60))
late.assertStatus(409)
;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers({ authorization: `Bearer ${second}` }).json({ error: 'test' })).assertStatus(200)
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'queued')
})
test('no leases while the database is over the disk limit', async ({ client }) => {
const token = await SeedOrchestrator.issueToken('full-disk')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const limit = indexerConfig.backfill.maxDefaultGb
indexerConfig.backfill.maxDefaultGb = 0
try {
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204)
} finally {
indexerConfig.backfill.maxDefaultGb = limit
}
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(200)
})
test('a range given back before it started does not use up an attempt', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('no-news-server')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range
;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers(auth).json({ error: 'NNTP login: 502', uncounted: true })).assertStatus(200)
const row = await db.from('seed_ranges').where('id', range.id).first()
assert.equal(row.status, 'queued')
assert.equal(row.attempts, 0)
})
test('relay: a leased range gets its overview chunks through the API', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('relay-node')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range
const fetch = SeedFetchPool.fetch
const connections = indexerConfig.seed.nntpConnections
const asked: unknown[] = []
SeedFetchPool.fetch = async (group, from, to) => {
asked.push([group, from, to])
return deflateSync(Buffer.from('5\tsubject\r\n'))
}
try {
indexerConfig.seed.nntpConnections = 0
;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 1, to: 50 }).headers(auth)).assertStatus(503)
indexerConfig.seed.nntpConnections = 1
const chunk = await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 50, to: 500 }).headers(auth)
chunk.assertStatus(200)
// Clamped to the range.
assert.deepEqual(asked, [['alt.binaries.test', 50, 100]])
;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 200, to: 300 }).headers(auth)).assertStatus(400)
} finally {
SeedFetchPool.fetch = fetch
indexerConfig.seed.nntpConnections = connections
}
})
test('ranges of a higher priority are leased first', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('priority-node')
await db.table('seed_ranges').insert([
{ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
{ group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101, priority: 1 },
])
const leased = await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })
assert.include(leased.body().range, { start: 101, end: 200 })
})
test('priorities take turns: half the leases go to the higher priority', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('turns-node')
const auth = { authorization: `Bearer ${token}` }
await db.table('seed_ranges').insert([
{ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
{ group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101 },
{ group_name: 'alt.binaries.test', start_id: 1001, end_id: 1100, next_id: 1001, priority: 1 },
{ group_name: 'alt.binaries.test', start_id: 1101, end_id: 1200, next_id: 1101, priority: 1 },
])
const starts: number[] = []
for (let i = 0; i < 4; i++) starts.push((await client.post('/api/v1/seed/lease').headers(auth)).body().range.start)
assert.deepEqual(starts, [1001, 1, 1101, 101])
})
test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [file] = await FileStore.insert([
{
filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar',
file_key: 'inspect-node-file'.padEnd(40, '0'),
poster: 'x',
date: now.getTime(),
parts: 2,
groups: '[]',
segments: [
{ number: 1, id: 'one@test', size: 100 },
{ number: 2, id: 'two@test', size: 100 },
],
release_id: release.id,
created_at: now,
updated_at: now,
},
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)
leased.assertStatus(200)
const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id)
assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test'])
// Locked: nobody else gets it meanwhile.
;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204)
// What probeInspection found in the first segment; posted as a JSON string, empty strings and all.
const result = {
size: 15_728_640,
contentType: null,
md5_16k: null,
yencName: '',
archive: {
format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120,
entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }],
},
}
const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) }
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200)
const row = await db.from('releases').where('id', release.id).first()
assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP')
assert.isTrue(row.encrypted)
assert.lengthOf(row.contents, 1)
assert.isOk((await db.from('file_details').where('file_id', file.id).first())?.archive)
assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
// Too late (or twice): the lock is gone.
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409)
})
test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node-2')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now },
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
const first = await lease()
;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200)
const again = await lease()
assert.equal(again.jobId, first.jobId)
;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: 'NNTP BODY: no answer within 120 s' }).headers(auth)).assertStatus(200)
// Retried later (backoff), like a local failure.
assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState())
})
test('a missing first article reported by a node is recorded, not retried', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node-3')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'Wq7n3LpX9vT2rK8mZ5cYbA.bin', file_key: 'inspect-node-file-3'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'gone@test', size: 1 }], created_at: now, updated_at: now },
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const job = (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
const error = 'Seed API /api/v1/seed/articles?id=gone%40test: 410 {"error":"No such article"}'
;(await client.post('/api/v1/seed/inspect/failed').json({ queue: job.queue, jobId: job.jobId, token: job.token, error }).headers(auth)).assertStatus(200)
assert.equal((await db.from('file_details').where('file_id', file.id).first())?.content_type, 'missing')
assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
})
test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('late-node')
const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id')
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204)
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done')
})
})
tests/functional/sfv.spec.ts
-59
View File
@@ -1,59 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import SfvService from '#services/SfvService'
import { parseSfv } from '#lib/content/index'
const POSTER = 'Snazzy Poster <me@localhost.localdomain>'
let keys = 0
async function release(name: string, source = 'stem') {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: `sfv${++keys}`.padEnd(40, 'x'), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
return Number(r.id)
}
async function file(name: string, releaseId: number, poster = POSTER) {
const [{ id }] = await FileStore.insert([
{ filename: name, poster, date: Date.now(), parts: 1, size: 10, release_id: releaseId, groups: '[]',
segments: [{ number: 1, id: `${name}@x`, size: 10 }], created_at: new Date(), updated_at: new Date() },
])
return db.from('files').where('id', id).first()
}
test.group('SFV grouping', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('parses SFV listings', ({ assert }) => {
assert.deepEqual(parseSfv('; made by x\r\n01-a-b-4066667a.flac 4066667A\r\n"02 c.flac" 5058F120\r\n\r\n'), [
{ name: '01-a-b-4066667a.flac', crc: '4066667a' },
{ name: '02 c.flac', crc: '5058f120' },
])
})
test("gathers an album's tracks into the SFV's release, named after the album; later tracks join too", async ({ assert }) => {
const album = await release('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026')
const sfv = await file('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026.sfv', album)
const track1 = await release('01-graham_barham-country_music-4066667a')
await file('01-graham_barham-country_music-4066667a.flac', track1)
// Another poster's file with the same name stays.
const other = await release('01-graham_barham-country_music-4066667a', 'stem')
const foreign = await file('01-graham_barham-country_music-4066667a.flac', other, 'someone@else')
const listing = '01-graham_barham-country_music-4066667a.flac 4066667a\n02-graham_barham-sippin-5058f120.flac 5058f120\n'
assert.equal(await SfvService.attach(sfv, listing), 1)
const renamed = await db.from('releases').where('id', album).first()
assert.equal(renamed.name, 'graham_barham-club_country_(after_party)-16bit-web-flac-2026')
assert.equal((await db.from('files').where('id', foreign.id).first()).release_id, other)
// Track 2 completes after the SFV was read.
const track2 = await release('02-graham_barham-sippin-5058f120')
const late = await file('02-graham_barham-sippin-5058f120.flac', track2)
assert.equal(await SfvService.claim({ id: late.id, name: late.filename, poster: late.poster, date: Number(late.date), release_id: track2 }), album)
assert.equal((await db.from('releases').where('id', album).first()).file_count, 3)
})
})
tests/functional/split.spec.ts
-84
View File
@@ -1,84 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import SplitService from '#services/SplitService'
import { weakFileKey } from '#lib/subject/index'
import { crc32Hex } from '#lib/yenc/index'
// Three 4-byte parts per volume, with real CRCs so the split volumes verify.
const DATA: Record<string, Buffer> = { A: Buffer.from('AAAAaaaa1111'), B: Buffer.from('BBBBbbbb2222') }
const KEY = weakFileKey('a8f3Kq', 10_000_952, 3)
function article(volume: string, number: number, articleNumber: number): IngestSegment {
return {
fileKey: KEY,
poster: `random-${articleNumber}@example.invalid`,
subject: '"a8f3Kq" yEnc',
name: `random${articleNumber}`,
totalParts: 3,
number,
messageId: `${volume}-${number}@test`,
bytes: 700_000,
articleNumber,
groups: ['alt.binaries.test'],
weakIdentity: true,
size: 10_000_952,
release: null,
pcrc32: crc32Hex(DATA[volume].subarray((number - 1) * 4, number * 4)),
partSize: 4,
fileCrc32: number === 3 ? crc32Hex(DATA[volume]) : undefined,
}
}
test.group('SplitService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.each.setup(() => {
// Keep completed volumes out of the real file queue.
const enqueue = IngestService.enqueue
IngestService.enqueue = async () => {}
return () => {
IngestService.enqueue = enqueue
}
})
test('splits two equal-sized volumes into verified, complete files', async ({ assert }) => {
// Volume A then volume B, arriving interleaved and out of order.
await IngestService.ingest([article('A', 1, 101), article('A', 2, 102), article('B', 1, 104)])
await IngestService.ingest([article('B', 2, 105), article('A', 3, 103), article('B', 3, 106)])
const parent = await db.from('binaries').where('file_key', KEY).first()
assert.isNotNull(parent.conflicted_at)
const result = await SplitService.splitPending()
assert.equal(result.volumes, 2)
assert.equal(result.completed, 2)
const volumes = await db.from('binaries').where('parent_binary_id', parent.id).orderBy('id')
assert.lengthOf(volumes, 2)
for (const [i, volume] of volumes.entries()) {
assert.isNotNull(volume.completed_at)
const parts = await db.from('parts').where('binary_id', volume.id).orderBy('number').select('message_id')
const letter = i === 0 ? 'A' : 'B'
assert.deepEqual(parts.map((p) => p.message_id), [1, 2, 3].map((n) => `${letter}-${n}@test`))
}
// Nothing new since the split: the next run leaves it alone.
assert.equal((await SplitService.splitPending()).parents, 0)
})
test('a mix that fails CRC verification is not turned into a file', async ({ assert }) => {
// B's last part is missing: the only complete candidate would mix volumes.
await IngestService.ingest([article('A', 1, 101), article('B', 1, 102), article('A', 2, 103)])
await IngestService.ingest([article('B', 2, 104), article('A', 3, 105)])
const parent = await db.from('binaries').where('file_key', KEY).first()
const result = await SplitService.splitPending()
const volumes = await db.from('binaries').where('parent_binary_id', parent.id)
for (const volume of volumes) {
const parts = await db.from('parts').where('binary_id', volume.id).select('message_id')
assert.lengthOf(new Set(parts.map((p) => p.message_id[0])), 1)
}
assert.isAtMost(result.completed, 1)
})
})
tests/functional/spot_listing.spec.ts
-193
View File
@@ -1,193 +0,0 @@
import { test } from '@japa/runner'
import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import SpotService, { isDummyNzb, NzbUnreadable } from '#services/SpotService'
import SearchService from '#services/SearchService'
test.group('SpotService listings', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('old verified spots become visible releases without files', async ({ assert }) => {
const now = new Date()
const old = new Date('2019-05-01T12:00:00Z')
const spot = {
message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
}
await db.table('spots').insert(spot)
const [claimed] = await SpotService.claim(10)
assert.isTrue(SpotService.isListable(claimed))
assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
assert.equal(await SpotService.list([claimed]), 1)
const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
assert.equal(row.status, 'listed')
const release = await db.from('releases').where('id', row.release_id).first()
assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
assert.equal(Number(release.size), 1_500_000_000)
// The key an import of this spot's NZB uses, so it fills this release.
assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
})
test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
// A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
const overviews = Array.from({ length: 6000 }, (_, i) => [
i + 1,
{ 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
]) as any
assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
assert.equal(Number(count), 6000)
})
test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
const spot = {
message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
details_at: new Date(),
}
await db.table('spots').insert(spot)
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
const release = (await SearchService.release(Number(releaseId)))!
assert.equal(release.spot!.description, 'Plot[br]more')
assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
assert.equal(release.spot!.categories.head, 'Image')
assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
})
test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
// Text-only spots announce the dummy's own size.
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
})
})
test.group('SpotService passwords', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
const now = new Date()
await db.table('spots').insert({
message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
})
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
await db.from('releases').where('id', releaseId).update({ encrypted: true })
const release = (await SearchService.release(Number(releaseId)))!
assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
// An NZB's own password is not replaced.
await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
await SpotService.applyPassword('pw1@spot.net')
assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
})
test('password states', async ({ assert }) => {
const { passwordState } = await import('#services/SearchService')
assert.equal(passwordState(true, false), 'missing')
assert.equal(passwordState(true, true), 'known')
assert.equal(passwordState(null, true), 'known')
assert.equal(passwordState(false, false), 'none')
assert.equal(passwordState(null, false), 'none')
})
})
test.group('SpotService reading listings', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
const nzb = (password: string | null, bytes = 700_000_000) => `<?xml version="1.0" encoding="UTF-8"?>
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb"><head>${password ? `<meta type="password">${password}</meta>` : ''}</head>
<file poster="p" date="1700000000" subject="&quot;a.part1.rar&quot; yEnc (1/1)"><groups><group>alt.binaries.test</group></groups>
<segments><segment bytes="${bytes}" number="1">a1@x</segment></segments></file></nzb>`
/** A listed spot, as worker:spots leaves an old one, with its article and NZB as given. */
async function listedSpot(id: string, read: { description?: string; nzb?: string | null; error?: string; nzbError?: Error }) {
await db.table('spots').insert({
message_id: id, group_name: 'free.pt', article_number: 1, title: `Listed ${id}`, poster: 'Bart', key_id: 7, size: 700_000_000,
posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: new Date(), status: 'pending',
})
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
const service = SpotService as any
service.readHead = async () => {
if (read.error) throw new Error(read.error)
return { header: { size: 700_000_000 }, xml: { description: read.description ?? '', nzbSegments: ['n@x'] } }
}
service.readNzb = async () => {
if (read.nzbError) throw read.nzbError
return read.nzb ?? null
}
}
const restore = () => {
delete (SpotService as any).readHead
delete (SpotService as any).readNzb
}
group.each.teardown(restore)
const passwordOf = async (id: string) => {
const spot = await db.from('spots').where('message_id', id).first()
const release = await db.from('releases').where('id', spot.release_id).first()
return { spot, password: release.password, source: release.password_source }
}
test("reads the NZB's password and the details, without importing files", async ({ assert }) => {
await listedSpot('l1@spot.net', { description: 'Plot', nzb: nzb('fromnzb1') })
assert.deepEqual(await SpotService.readListings(10, 2), { 'nzb-password': 1 })
const { spot, password, source } = await passwordOf('l1@spot.net')
assert.deepEqual([password, source, spot.status, spot.description], ['fromnzb1', 'nzb', 'listed', 'Plot'])
assert.isNotNull(spot.nzb_read_at)
assert.equal((await db.from('releases').where('id', spot.release_id).first()).file_count, 0)
// Read once: nothing left.
assert.deepEqual(await SpotService.readListings(10, 2), {})
})
test("the spot's text when the NZB has none; the NZB's own password wins over it", async ({ assert }) => {
await listedSpot('l2@spot.net', { description: 'Wachtwoord: fromtext2', nzb: nzb(null) })
assert.deepEqual(await SpotService.readListings(10, 1), { 'spot-password': 1 })
assert.include(await passwordOf('l2@spot.net'), { password: 'fromtext2', source: 'spot' })
await listedSpot('l3@spot.net', { description: 'Wachtwoord: fromtext3', nzb: nzb('fromnzb3') })
await db.from('releases').where('id', (await db.from('spots').where('message_id', 'l3@spot.net').first()).release_id).update({ password: 'fromtext3', password_source: 'spot' })
assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-password': 1 })
assert.include(await passwordOf('l3@spot.net'), { password: 'fromnzb3', source: 'nzb' })
})
test('a placeholder NZB hides the listing; a lost connection gives the spot back, a gone article does not', async ({ assert }) => {
await listedSpot('l4@spot.net', { nzb: nzb(null, 1500) })
assert.deepEqual(await SpotService.readListings(10, 1), { dummy: 1 })
assert.equal((await db.from('spots').where('message_id', 'l4@spot.net').first()).status, 'dummy')
await listedSpot('l5@spot.net', { error: 'Connection reset' })
assert.deepEqual(await SpotService.readListings(10, 1), { retry: 1 })
assert.isNull((await db.from('spots').where('message_id', 'l5@spot.net').first()).nzb_read_at)
await db.from('spots').where('message_id', 'l5@spot.net').delete()
await listedSpot('l6@spot.net', { error: '430 No Such Article' })
assert.deepEqual(await SpotService.readListings(10, 1), { gone: 1 })
assert.isNotNull((await db.from('spots').where('message_id', 'l6@spot.net').first()).nzb_read_at)
// A corrupt NZB is a result, not something to read again.
await listedSpot('l7@spot.net', { nzbError: new NzbUnreadable('invalid code lengths set') })
assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-unreadable': 1 })
const l7 = await db.from('spots').where('message_id', 'l7@spot.net').first()
assert.isNotNull(l7.nzb_read_at)
assert.equal(l7.error, 'NZB unreadable: invalid code lengths set')
})
})
tests/functional/staging.spec.ts
-70
View File
@@ -1,70 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import StagingService from '#services/StagingService'
import IngestService, { segmentFromHeader } from '#services/IngestService'
const segment = (name: string, k: number, n: number, id = `${name}-${k}@x`) =>
segmentFromHeader(
{ 'subject': `"${name}" yEnc (${k}/${n})`, 'from': 'poster@x', 'message-id': `<${id}>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
'alt.binaries.test',
k
)!
async function binary(name: string) {
return db.from('binaries').where('name', name).first()
}
test.group('Staging', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a lone segment waits; its second segment brings both in', async ({ assert }) => {
let r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
assert.equal(r.staged, 1)
assert.lengthOf(r.ingest, 0)
await IngestService.ingest(r.ingest)
assert.isNull(await binary('rnd1.bin'))
// The same article again (a re-fetched range) is no second segment.
r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
assert.lengthOf(r.ingest, 0)
// Nor is the same article crossposted to another group (other group, other article number).
const crosspost = { ...segment('rnd1.bin', 3, 50), groups: ['alt.binaries.boneless'], articleNumber: 160154327571 }
r = await StagingService.filter([crosspost])
assert.lengthOf(r.ingest, 0)
r = await StagingService.filter([segment('dup.bin', 2, 9), { ...segment('dup.bin', 2, 9), groups: ['alt.binaries.boneless'] }])
assert.equal(r.staged, 1)
r = await StagingService.filter([segment('rnd1.bin', 4, 50)])
assert.deepEqual(r.ingest.map((s) => s.number).sort(), [3, 4])
await IngestService.ingest(r.ingest)
const b = await binary('rnd1.bin')
assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 2)
assert.lengthOf(await db.from('staged_segments').where('file_key', segment('rnd1.bin', 3, 50).fileKey), 0)
// Later segments of a known file go straight in.
r = await StagingService.filter([segment('rnd1.bin', 9, 50)])
assert.lengthOf(r.ingest, 1)
})
test('single-part files and files with two segments in the batch go straight in; staged ones join them', async ({ assert }) => {
// Part 1 arrived alone at the end of the previous batch.
await StagingService.filter([segment('real.part01.rar', 1, 3)])
const r = await StagingService.filter([segment('real.part01.rar', 2, 3), segment('real.part01.rar', 3, 3), segment('one.nfo', 1, 1)])
assert.equal(r.staged, 0)
assert.sameMembers(r.ingest.map((s) => `${s.name}:${s.number}`), ['real.part01.rar:1', 'real.part01.rar:2', 'real.part01.rar:3', 'one.nfo:1'])
})
test('GC adopts segments whose file got a binary, and drops old ones', async ({ assert }) => {
await StagingService.filter([segment('late.bin', 7, 10)])
await StagingService.filter([segment('old.bin', 2, 10)])
await db.from('staged_segments').where('file_key', segment('old.bin', 2, 10).fileKey).update({ staged_at: new Date(Date.now() - 3 * 3600_000) })
// Another job created the binary meanwhile.
await IngestService.ingest([segment('late.bin', 1, 10), segment('late.bin', 2, 10)])
const result = await StagingService.sweep(2 * 3600, Date.now() + 10_000)
assert.deepEqual(result, { adopted: 1, expired: 1 })
const b = await binary('late.bin')
assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 3)
})
})
tests/functional/web_api.spec.ts
-145
View File
@@ -1,145 +0,0 @@
import { test } from '@japa/runner'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import testUtils from '@adonisjs/core/services/test_utils'
async function seed() {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'w'.repeat(40), name: 'Webtest.Show.S01E02.1080p', source: 'par2', groups: '["alt.binaries.test"]', posted_at: now, first_seen_at: now, last_seen_at: now, contents: JSON.stringify([{ name: 'Webtest.Show.S01E02.1080p/episode.mkv', size: 1000 }]) })
.returning('id')
await FileStore.insert([{
filename: 'Xq8r2KdLm3Zw91PqA7webtest', real_name: 'Webtest.Show.S01E02.1080p.mkv', poster: 'p@example.invalid',
date: now.getTime(), parts: 2, size: 1400, release_id: release.id, groups: '["alt.binaries.test"]',
segments: [{ number: 1, id: 'one@test', size: 700 }, { number: 2, id: 'two@test', size: 700 }],
created_at: now, updated_at: now,
}])
return Number(release.id)
}
test.group('Web API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('searches by words in any order, with dots and spaces interchangeable', async ({ client, assert }) => {
const id = await seed()
for (const q of ['webtest show s01e02', 'Webtest.Show', 'episode.mkv 1080p']) {
const response = await client.get('/api/v1/releases').qs({ q })
response.assertStatus(200)
assert.include(response.body().releases.map((r: any) => r.id), id, q)
}
const none = await client.get('/api/v1/releases').qs({ q: 'webtest nosuchword' })
assert.notInclude(none.body().releases.map((r: any) => r.id), id)
})
test('filters and details', async ({ client, assert }) => {
const id = await seed()
const readable = await client.get('/api/v1/releases').qs({ q: 'webtest', readable: '1', source: 'par2' })
assert.include(readable.body().releases.map((r: any) => r.id), id)
const other = await client.get('/api/v1/releases').qs({ q: 'webtest', source: 'rar' })
assert.notInclude(other.body().releases.map((r: any) => r.id), id)
const detail = await client.get(`/api/v1/releases/${id}`)
detail.assertStatus(200)
assert.equal(detail.body().files[0].name, 'Webtest.Show.S01E02.1080p.mkv')
assert.equal(detail.body().fileCount, 1)
assert.equal(detail.body().size, 1400)
;(await client.get('/api/v1/releases/999999999')).assertStatus(404)
})
test('filters by size in MB and by file extension as a whole word', async ({ client, assert }) => {
const id = await seed() // 1400 bytes, an episode.mkv in its listing
const ids = async (qs: Record<string, string>) =>
(await client.get('/api/v1/releases').qs({ q: 'webtest', ...qs })).body().releases.map((r: any) => r.id)
assert.include(await ids({ min_size: '0.001', max_size: '0.002' }), id)
assert.notInclude(await ids({ min_size: '1' }), id)
assert.notInclude(await ids({ max_size: '0.001' }), id)
assert.include(await ids({ ext: 'mkv' }), id)
assert.include(await ids({ ext: '.iso, MKV' }), id)
assert.notInclude(await ids({ ext: 'iso' }), id)
// "sho" is part of "show", not an extension.
assert.notInclude(await ids({ ext: 'sho' }), id)
// A plainly posted file counts too, not only archive entries; volumes count as their archive.
const now = new Date()
await FileStore.insert([
{ filename: 'Webtest.Show.S01E02.1080p.nfo', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now },
{ filename: 'Webtest.Show.S01E02.1080p.7z.002', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now },
])
assert.include(await ids({ ext: 'nfo' }), id)
assert.include(await ids({ ext: '7z' }), id)
assert.deepEqual((await db.from('releases').where('id', id).first()).extensions, ['7z', 'mkv', 'nfo'])
})
test('serves an NZB with the real file name', async ({ client, assert }) => {
const id = await seed()
const response = await client.get(`/api/v1/releases/${id}/nzb`)
response.assertStatus(200)
assert.include(response.header('content-type'), 'application/x-nzb')
assert.include(response.header('content-disposition'), 'Webtest.Show.S01E02.1080p.nzb')
assert.include(response.text(), '&quot;Webtest.Show.S01E02.1080p.mkv&quot; yEnc (1/2)')
assert.include(response.text(), '<segment bytes="700" number="1">one@test</segment>')
})
test('stats', async ({ client }) => {
const response = await client.get('/api/v1/stats')
response.assertStatus(200)
response.assertBodyContains({ sources: [] })
})
test('completeness follows files and par2 lists (triggers note releases, the sweep recomputes)', async ({ assert }) => {
const now = new Date()
const complete = async (id: number) => {
await ReleaseCompleteness.sweep()
return (await db.from('releases').where('id', id).first()).complete
}
const file = (releaseId: number, name: string) => ({
filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
groups: '[]', segments: [], created_at: now, updated_at: now,
})
// Counter rule: [i/M] announced 2 files.
const [counted] = await db.table('releases').insert({ release_key: 'c'.repeat(40), name: 'Counted', groups: '[]', file_total: 2, first_seen_at: now, last_seen_at: now }).returning('id')
await FileStore.insert([file(counted.id, 'a.rar')])
assert.isFalse(await complete(counted.id))
await FileStore.insert([file(counted.id, 'b.rar')])
assert.isTrue(await complete(counted.id))
// par2 rule: the listed file becomes present once a file of its exact size is there.
const [listed] = await db.table('releases').insert({ release_key: 'l'.repeat(40), name: 'Listed', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id')
await db.table('release_files').insert({ release_id: listed.id, par2_file_id: '1'.repeat(32), name: 'Listed.mkv', size: 5_000_000, md5_16k: '0'.repeat(32) })
assert.isFalse(await complete(listed.id))
const [f] = await FileStore.insert([file(listed.id, 'Xq8r2KdLm3Zw91PqA7')])
const [b] = await db.table('binaries').insert({ file_key: 'z'.repeat(40), poster: 'p', subject: '"x" yEnc', name: 'x', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, file_id: f.id }).returning('id')
assert.isFalse(await complete(listed.id))
await db.from('binaries').where('id', b.id).update({ size: 5_000_000 })
assert.isTrue(await complete(listed.id))
})
test('par2-only releases are hidden unless asked for', async ({ client, assert }) => {
const now = new Date()
const [release] = await db.table('releases')
.insert({ release_key: 'p'.repeat(40), name: 'Webtest.Par2only.S01E01.1080p.WEB-DL-GRP', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
await db.table('release_files').insert({ release_id: release.id, par2_file_id: '2'.repeat(32), name: 'Webtest.Par2only.S01E01.mkv', size: 1_000_000_000, md5_16k: '0'.repeat(32) })
await FileStore.insert([{ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
release_id: release.id, groups: '[]', segments: [], created_at: now, updated_at: now }])
const hidden = await client.get('/api/v1/releases').qs({ q: 'webtest par2only' })
assert.notInclude(hidden.body().releases.map((r: any) => r.id), Number(release.id))
const shown = await client.get('/api/v1/releases').qs({ q: 'webtest par2only', par2_only: '1' })
const row = shown.body().releases.find((r: any) => r.id === Number(release.id))
assert.isTrue(row.par2Only)
})
})
test.group('Health check', () => {
test('/health answers when the database does', async ({ client }) => {
const response = await client.get('/health')
response.assertStatus(200)
response.assertBody({ ok: true })
})
})
tests/functional/yenc_name.spec.ts
-40
View File
@@ -1,40 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import ContentService from '#services/ContentService'
let keys = 0
async function counterFile(name: string) {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: `yenc${++keys}`.padEnd(40, 'y'), name, source: 'weak-stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [{ id }] = await FileStore.insert([
{ filename: name, poster: 'bob@home.com', date: now.getTime(), parts: 2, size: 10, release_id: r.id, groups: '["alt.binaries.boneless"]',
segments: [{ number: 1, id: `${name}@nyuu`, size: 10 }], created_at: now, updated_at: now },
])
return db.from('files').where('id', id).first()
}
test.group('Names from the yEnc header', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('random-named files take the yEnc name and gather in its release', async ({ assert }) => {
const a = await counterFile('P9BSPl59iDfQUVvXFeBVG4MDQaOTufds')
const b = await counterFile('Faoq6nlEaVlTzYyNA27sSfFm2KvLHOvD')
const nameFromYenc = (ContentService as any).nameFromYenc.bind(ContentService)
assert.equal(await nameFromYenc(a, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar'), 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar')
await nameFromYenc(b, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part02.rar')
const [fa, fb] = await Promise.all([db.from('files').where('id', a.id).first(), db.from('files').where('id', b.id).first()])
assert.equal(fa.release_id, fb.release_id)
assert.equal((await db.from('releases').where('id', fa.release_id).first()).name, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP')
// A random yEnc name, or a file that has a real name already, stays as it is.
const c = await counterFile('Vg9QflfNLXVMPOWtK4KRJv474DadgTIF')
assert.isUndefined(await nameFromYenc(c, null, 'qTcrIl1qTo4Bx6El5c4RtTA4ZSqwd6tz'))
const d = await counterFile('Some.Release.part01.rar')
assert.isUndefined(await nameFromYenc(d, null, 'Other.Name.part01.rar'))
})
})