daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: fresh AdonisJS 7 API app on Postgres

Browse Source 
The rebuild branch starts over (REBUILD.md): the API starter kit, with
Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a
shared one in development) and the #lib alias. Phase 1's code is
imported from the current app module by module.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:05:24 +02:00
1 parent 699395c51f
commit 96f0ade10e
264 files changed
+777 -24132
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/bootstrap.ts
+1 -7
View File
@@ -42,13 +42,7 @@ export const plugins: Config['plugins'] = [
*/
export const runnerHooks: Required<Pick<Config, 'setup' | 'teardown'>> = {
setup: [],
teardown: [
// Tests that complete a file queue jobs; their Redis connections would keep the process alive.
async () => {
const { default: QueueService } = await import('#services/QueueService')
await QueueService.closeAll()
},
],
teardown: [],
}
/**
tests/functional/backfill.spec.ts
-38
View File
@@ -1,38 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import indexerConfig from '#config/indexer'
import BackfillService from '#services/BackfillService'
import FetchRangeService from '#services/FetchRangeService'
test.group('BackfillService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('queues header-only ranges downwards, capped, without holding back the live watermark', async ({ assert }) => {
const now = new Date()
const batch = Number(indexerConfig.batchSize)
const [g] = await db
.table('groups')
.insert({
name: 'alt.binaries.backfilltest', active: true, fetch_bodies: true, last_queued_id: 5_000_000, last_indexed_id: 5_000_000,
backfill_until_id: 5_000_000 - 100 * batch, backfill_next_id: 4_900_000, created_at: now, updated_at: now,
})
.returning('*')
const queued = await BackfillService.tick()
assert.equal(queued, indexerConfig.backfill.maxQueued)
const ranges = await db.from('fetch_ranges').where('group_id', g.id).orderBy('end_id', 'desc')
assert.isTrue(ranges.every((r) => r.backfill && r.status === 'queued'))
assert.equal(Number(ranges[0].end_id), 4_900_000)
assert.equal(Number(ranges[0].start_id), 4_900_000 - batch + 1)
const after = await db.from('groups').where('id', g.id).first()
assert.equal(Number(after.backfill_next_id), 4_900_000 - queued * batch)
// The cap holds while those wait.
assert.equal(await BackfillService.tick(), 0)
// Backfill ranges don't hold back the live watermark.
await db.from('groups').where('id', g.id).update({ last_queued_id: 5_100_000 })
assert.equal(await FetchRangeService.advanceWatermark({ id: g.id, name: g.name }), 5_100_000n)
})
})
tests/functional/binary_files.spec.ts
-86
View File
@@ -1,86 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import BinaryFileService from '#services/BinaryFileService'
import QueueService from '#services/QueueService'
import FileStore, { segmentsOf } from '#services/FileStore'
import { fileKey } from '#lib/subject/index'
import ContentService from '#services/ContentService'
function segment(name: string, number: number, total = 3): IngestSegment {
const poster = 'batch-test@example.invalid'
const subject = `"${name}" yEnc`
return { fileKey: fileKey(poster, subject, total), poster, subject, name, totalParts: total, number, messageId: `${name}-${number}@test`, bytes: 100, groups: ['alt.binaries.test'] }
}
const binary = (name: string, total = 3) => db.from('binaries').where('file_key', segment(name, 1, total).fileKey).first()
test.group('Binaries to files', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
test('ingest counts stored parts, once each, and claims a binary by that count', async ({ assert }) => {
await IngestService.ingest([segment('a.bin', 1), segment('a.bin', 2)])
// The same article again (a crosspost): not counted twice.
await IngestService.ingest([segment('a.bin', 2)])
assert.equal((await binary('a.bin')).part_count, 2)
assert.isNull((await binary('a.bin')).completed_at)
const completed = await IngestService.ingest([segment('a.bin', 3)])
assert.equal((await binary('a.bin')).part_count, 3)
assert.deepEqual(completed.map((c) => c.fileKey), [segment('a.bin', 1).fileKey])
})
test('a binary from before the count is counted exactly on its next part', async ({ assert }) => {
await IngestService.ingest([segment('b.bin', 1), segment('b.bin', 2)])
await db.from('binaries').where('id', (await binary('b.bin')).id).update({ part_count: null })
const completed = await IngestService.ingest([segment('b.bin', 3)])
assert.equal((await binary('b.bin')).part_count, 3)
assert.lengthOf(completed, 1)
})
test('stores a batch: new files, existing ones linked, incomplete ones left for the retry', async ({ assert }) => {
await IngestService.ingest([1, 2, 3].flatMap((n) => [segment('new.bin', n), segment('known.bin', n)]))
await IngestService.ingest([segment('half.bin', 1)])
// known.bin is a file already (e.g. from an NZB), found by its first segment.
const now = new Date()
const [known] = await FileStore.insert([
{ filename: 'known.bin', poster: 'x', date: now.getTime(), parts: 3, groups: '[]', segments: [1, 2, 3].map((n) => ({ number: n, id: `known.bin-${n}@test`, size: 100 })), created_at: now, updated_at: now },
])
const ids = [await binary('new.bin'), await binary('known.bin'), await binary('half.bin')].map((b) => Number(b.id))
const result = await BinaryFileService.store(ids)
assert.include(result, { saved: 1, existing: 1 })
assert.deepEqual(result.incomplete, [ids[2]])
const stored = await db.from('files').where('file_key', segment('new.bin', 1).fileKey).first()
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
assert.equal(Number((await binary('new.bin')).file_id), Number(stored.id))
assert.equal(Number((await binary('known.bin')).file_id), known.id)
// Header parts are in the file's segment list now.
const [{ n }] = await db.from('parts').whereIn('binary_id', [ids[0], ids[1]]).count('* as n')
assert.equal(Number(n), 0)
// The retry: done ones are skipped, the incomplete one again incomplete.
const again = await BinaryFileService.store(ids)
assert.include(again, { saved: 0, skipped: 2 })
assert.deepEqual(again.incomplete, [ids[2]])
})
test('a queued obfuscated file keeps its binary until inspection fingerprints it', async ({ assert }) => {
const name = 'Zk3m9QpL2xW8rT5vN1bYq.bin'
await IngestService.ingest([1, 2, 3].map((n) => segment(name, n)))
const b = await binary(name)
const result = await BinaryFileService.store([Number(b.id)])
assert.include(result, { saved: 1, inspections: 1 })
const waiting = await binary(name)
assert.isTrue(waiting.awaiting_inspection)
// Inspection stores the fingerprint and clears the flag.
const task = { kind: 'inspect' as const, fileId: Number(waiting.file_id), name, segments: [], sevenZip: null }
await ContentService.apply(task, { size: 300, contentType: null, md5_16k: 'a'.repeat(32), yencName: name })
const inspected = await binary(name)
assert.isNull(inspected.awaiting_inspection)
assert.equal(inspected.md5_16k, 'a'.repeat(32))
})
})
tests/functional/counter_grouping.spec.ts
-70
View File
@@ -1,70 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import StagingService from '#services/StagingService'
import IngestService, { segmentFromHeader } from '#services/IngestService'
import { counterFileKey } from '#lib/subject/index'
let n = 0
/** An article of a random-subject upload: a new random name every time. */
const randomArticle = (k: number, total = 4, { counter = '[21/62]', date = 'Thu, 01 Oct 2026 12:00:00 GMT', id }: { counter?: string; date?: string; id?: string } = {}) =>
segmentFromHeader(
{ 'subject': `${counter} "Rnd${++n}xQ7kLm2Pz9Wv4Ty8Ab3Cd" yEnc (${k}/${total})`, 'from': 'bob@home.com', 'message-id': `<${id ?? `art${n}`}@nyuu>`, 'date': date, ':bytes': '700000' },
'alt.binaries.boneless',
k
)!
async function ingestBatch(segments: ReturnType<typeof randomArticle>[], counters = true) {
const r = await StagingService.filter(segments, { counters })
await IngestService.ingest(r.ingest)
return r
}
test.group('Counter grouping', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('articles with random subjects become one file by poster, counter and part count', async ({ assert }) => {
const key = counterFileKey(randomArticle(1))!
await ingestBatch([randomArticle(1)])
assert.isNull(await db.from('binaries').where('file_key', key).first())
await ingestBatch([randomArticle(2)])
const binary = await db.from('binaries').where('file_key', key).first()
assert.isNotNull(binary)
// Later articles join the file directly, until it completes.
await ingestBatch([randomArticle(3)])
await ingestBatch([randomArticle(4)])
const parts = await db.from('parts').where('binary_id', binary.id).orderBy('number').select('number')
assert.deepEqual(parts.map((p) => p.number), [1, 2, 3, 4])
assert.isNotNull((await db.from('binaries').where('id', binary.id).first()).completed_at)
})
test('off by default; another day is another file; a repeated part number is a conflict', async ({ assert }) => {
await ingestBatch([randomArticle(1, 9, { counter: '[05/10]' })], false)
await ingestBatch([randomArticle(2, 9, { counter: '[05/10]' })], false)
assert.isNull(await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[05/10]' }))!).first())
const other = randomArticle(1, 9, { counter: '[06/10]', date: 'Fri, 02 Oct 2026 12:00:00 GMT' })
assert.notEqual(counterFileKey(other), counterFileKey(randomArticle(1, 9, { counter: '[06/10]' })))
// Two uploads of the same shape on one day: part 1 twice, from different articles.
await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
await ingestBatch([randomArticle(2, 9, { counter: '[07/10]' })])
await ingestBatch([randomArticle(1, 9, { counter: '[07/10]' })])
const binary = await db.from('binaries').where('file_key', counterFileKey(randomArticle(1, 9, { counter: '[07/10]' }))!).first()
assert.isNotNull(binary.conflicted_at)
})
test('a normal file with a counter stays a normal file', async ({ assert }) => {
const normal = (k: number) =>
segmentFromHeader(
{ 'subject': `[07/45] - "Some.Release.part06.rar" yEnc (${k}/3)`, 'from': 'poster@x', 'message-id': `<norm${k}@x>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
'alt.binaries.test',
k
)!
await ingestBatch([normal(1)])
await ingestBatch([normal(2), normal(3)])
const binary = await db.from('binaries').where('name', 'Some.Release.part06.rar').first()
assert.equal(binary.file_key, normal(1).fileKey)
assert.isNotNull(binary.completed_at)
})
})
tests/functional/deferred.spec.ts
-67
View File
@@ -1,67 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import DeferredService, { deferredName, isDeferred } from '#services/DeferredService'
const header = (id: string, name: string, extra: Record<string, string> = {}) => ({
'message-id': `<${id.padEnd(32, '0')}@ngPost>`,
'subject': `"${name}" yEnc`,
'from': 'pKeUimunULJZz@ngPost.com',
':bytes': '740000',
'date': 'Mon, 28 Sep 2026 10:00:00 +0000',
...extra,
})
test.group('DeferredService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('recognises deferred uploads by message-id and a quoted name', ({ assert }) => {
assert.equal(deferredName('[026/109] "HFIX7fbQAuH4PZsvGMw7Pz.part025.rar" yEnc'), 'HFIX7fbQAuH4PZsvGMw7Pz.part025.rar')
assert.isNull(deferredName('no quotes here'))
assert.isTrue(isDeferred(header('ab', 'x.part1.rar')))
assert.isFalse(isDeferred({ ...header('ab', 'x'), 'message-id': '<part1of5.abc@nyuu>' }))
})
test('articles without a name in the subject are kept as article ranges', async ({ assert }) => {
const hex = (id: string, n: number) => ({ articleNumber: n, header: { ...header(id, 'x'), subject: id.padEnd(32, 'f') } })
const files = await DeferredService.record('alt.binaries.test', [hex('a', 100), hex('b', 180), hex('c', 90_000), hex('d', 150)])
assert.equal(files, 0)
const ranges = await db.from('deferred_ranges').orderBy('first_article')
assert.deepEqual(
ranges.map((r) => [Number(r.first_article), Number(r.last_article), r.article_count]),
[[100, 180, 3], [90_000, 90_000, 1]]
)
assert.lengthOf(await db.from('deferred_files'), 0)
})
test('aggregates articles per file, across batches, counting only the first group', async ({ assert }) => {
await DeferredService.record('alt.binaries.test', [
{ articleNumber: 105, header: header('b', 'a.part1.rar') },
{ articleNumber: 101, header: header('a', 'a.part1.rar') },
{ articleNumber: 102, header: header('c', 'b.par2', { ':bytes': '1000' }) },
])
await DeferredService.record('alt.binaries.test', [{ articleNumber: 110, header: header('d', 'a.part1.rar') }])
// A crosspost seen in another group doesn't count twice.
await DeferredService.record('alt.binaries.misc', [{ articleNumber: 9, header: header('a', 'a.part1.rar') }])
const rows = await db.from('deferred_files').orderBy('name')
assert.lengthOf(rows, 2)
const [a, b] = rows
assert.include(a, { name: 'a.part1.rar', group_name: 'alt.binaries.test', article_count: 3, status: 'pending' })
assert.equal(Number(a.first_article), 101)
assert.equal(Number(a.last_article), 110)
assert.equal(a.first_message_id, 'a'.padEnd(32, '0') + '@ngPost')
assert.equal(Number(a.bytes), 3 * 740000)
assert.include(b, { name: 'b.par2', article_count: 1 })
})
test('only claims files that stopped growing', async ({ assert }) => {
await DeferredService.record('alt.binaries.test', [{ articleNumber: 1, header: header('a', 'new.rar') }])
assert.lengthOf(await DeferredService.claim(10), 0)
await db.from('deferred_files').update({ last_seen_at: new Date(Date.now() - 24 * 60 * 60 * 1000) })
const claimed = await DeferredService.claim(10)
assert.lengthOf(claimed, 1)
assert.equal(claimed[0].status, 'processing')
assert.lengthOf(await DeferredService.claim(10), 0)
})
})
tests/functional/fetch_ranges.spec.ts
-46
View File
@@ -1,46 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FetchRangeService from '#services/FetchRangeService'
async function groupWithRanges(ranges: [number, number, string][]) {
const now = new Date()
const [group] = await db
.table('groups')
.insert({ name: `test.ranges.${Date.now()}`, active: false, last_indexed_id: 100, last_queued_id: 700, created_at: now, updated_at: now })
.returning(['id', 'name'])
for (const [start, end, status] of ranges) {
await db.table('fetch_ranges').insert({ group_id: group.id, start_id: start, end_id: end, status, created_at: now, updated_at: now })
}
return { id: Number(group.id), name: group.name }
}
test.group('FetchRangeService.advanceWatermark', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('stops just below the first range that is not done', async ({ assert }) => {
const g = await groupWithRanges([
[101, 200, 'done'],
[201, 300, 'failed'],
[301, 400, 'done'],
[401, 700, 'queued'],
])
assert.equal(await FetchRangeService.advanceWatermark(g), 200n)
// done ranges are forgotten, the gap and the queued range stay
const left = await db.from('fetch_ranges').where('group_id', g.id).orderBy('start_id').select('status')
assert.deepEqual(left.map((r) => r.status), ['failed', 'queued'])
})
test('reaches the queue cursor when everything is done', async ({ assert }) => {
const g = await groupWithRanges([
[101, 400, 'done'],
[401, 700, 'done'],
])
assert.equal(await FetchRangeService.advanceWatermark(g), 700n)
})
test('never moves backwards', async ({ assert }) => {
const g = await groupWithRanges([[50, 60, 'failed']])
assert.equal(await FetchRangeService.advanceWatermark(g), 100n)
})
})
tests/functional/file_keys.spec.ts
-41
View File
@@ -1,41 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileKeys from '#services/FileKeys'
import FileStore from '#services/FileStore'
const file = (key: string, name: string) => {
const now = new Date()
return { file_key: key, filename: name, poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [], created_at: now, updated_at: now }
}
test.group('file_keys', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.each.teardown(() => FileKeys.forget())
test('the fill copies keys from before, then file inserts use the filter', async ({ assert }) => {
// A file from before file_keys: in files, not in file_keys.
const [old] = await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'old.bin')])
await db.rawQuery(`DELETE FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-old'.padEnd(40, '0')])
await db.from('file_keys_fill').where('id', 1).update({ next_id: 0, until_id: null, complete: false })
FileKeys.forget()
// Not complete: every partition is checked, so the old key is found anyway.
assert.isFalse(await FileKeys.complete())
assert.lengthOf(await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'again.bin')]), 0)
const filled = await FileKeys.fill(60_000)
assert.isTrue(filled.complete)
assert.isAtLeast(filled.untilId, old.id)
const [{ n }] = (await db.rawQuery(`SELECT count(*) AS n FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-old'.padEnd(40, '0')])).rows
assert.equal(Number(n), 1)
// Complete: the filter says the old key may exist, the partitions confirm it; a new key goes in.
assert.isTrue(await FileKeys.complete())
assert.lengthOf(await FileStore.insert([file('fk-old'.padEnd(40, '0'), 'again.bin')]), 0)
assert.lengthOf(await FileStore.insert([file('fk-new'.padEnd(40, '0'), 'new.bin')]), 1)
// And the trigger put the new one in file_keys.
const [{ m }] = (await db.rawQuery(`SELECT count(*) AS m FROM file_keys WHERE key_hash = hashtextextended(?, 0)`, ['fk-new'.padEnd(40, '0')])).rows
assert.equal(Number(m), 1)
})
})
tests/functional/file_partitions.spec.ts
-51
View File
@@ -1,51 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import FilePartitions, { FILE_PARTITION_SIZE } from '#services/FilePartitions'
const file = (fileKey: string, name: string) => {
const now = new Date()
return { file_key: fileKey, filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], created_at: now, updated_at: now }
}
test.group('files partitions', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
// Sequences ignore rollbacks: put files_id_seq back after each test.
group.each.setup(async () => {
const { rows } = await db.rawQuery(`SELECT last_value, is_called FROM files_id_seq`)
return async () => {
await db.rawQuery(`SELECT setval('files_id_seq', ?, ?)`, [rows[0].last_value, rows[0].is_called])
}
})
test('a file_key already in an older partition is not inserted again', async ({ assert }) => {
await db.rawQuery(`SELECT setval('files_id_seq', 500000, false)`)
const [first] = await FileStore.insert([file('k'.repeat(40), 'old.bin')])
// The next ids fall in the second partition, whose own unique index doesn't know the key.
await db.rawQuery(`SELECT setval('files_id_seq', 1000000, false)`)
const again = await FileStore.insert([file('k'.repeat(40), 'again.bin'), file('n'.repeat(40), 'new.bin'), file('n'.repeat(40), 'twice.bin')])
assert.deepEqual(again.map((f) => f.filename), ['new.bin'])
assert.isAtLeast(again[0].id, 1000000)
assert.equal(await db.from('files').where('file_key', 'k'.repeat(40)).count('* as n').then((r) => Number(r[0].n)), 1)
assert.isBelow(first.id, 1000000)
})
test('partitions are kept ready ahead of the newest id', async ({ assert }) => {
const upper = async () => {
const { rows } = await db.rawQuery(
`SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass`
)
return Number(rows[0].upper)
}
await FilePartitions.ensure()
const before = await upper()
assert.deepEqual(await FilePartitions.ensure(), [])
// Ids handed out up to just inside the four partitions kept ahead: one more is needed.
await db.rawQuery(`SELECT setval('files_id_seq', ?, false)`, [before - 4 * FILE_PARTITION_SIZE + 10])
await FileStore.insert([file('e'.repeat(40), 'late.bin')])
assert.deepEqual(await FilePartitions.ensure(), [`files_p${before}`])
assert.equal(await upper(), before + FILE_PARTITION_SIZE)
})
})
tests/functional/garbage.spec.ts
-190
View File
@@ -1,190 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import indexerConfig from '#config/indexer'
import FileStore from '#services/FileStore'
import SeedService from '#services/SeedService'
import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
import ContentService from '#services/ContentService'
import GarbageFilter from '#services/GarbageFilter'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toSeedFile } from '#lib/seed/wire'
import { explored, uploadTool } from '#lib/garbage/index'
const GROUP = 'alt.binaries.test'
let n = 0
/** Headers of a file posted by ngPost (or nyuu), all its parts. */
function upload(name: string, parts: number, tool: 'ngPost' | 'nyuu' = 'ngPost') {
const id = () => {
n++
return tool === 'ngPost' ? `${n.toString(16).padStart(32, '0')}@ngPost` : `${'FmFwIaTpGgUiSfYcMwFl'}${String.fromCharCode(65 + (n % 26))}e${String.fromCharCode(65 + ((n >> 5) % 26))}o-${String(1790637260000 + n)}@nyuu`
}
return Array.from({ length: parts }, (_, i) =>
segmentFromHeader(
{
'subject': `[1/1] - "${name}" yEnc (${i + 1}/${parts})`,
'from': 'poster <poster@example.invalid>',
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
'message-id': `<${id()}>`,
':bytes': '700000',
},
GROUP,
n
)!
)
}
async function dropSignature(tool: string, group = GROUP) {
await db.table('upload_signatures').insert({ tool, group_name: group, probed: 500, named: 0, dropped: true })
GarbageFilter.reset()
}
/** A random-named release with one stored file, as seeding leaves it. */
async function storedRelease(release: Record<string, unknown> = {}, file: Record<string, unknown> = {}) {
const now = new Date()
n++
const [r] = await db
.table('releases')
.insert({ release_key: `garbage${n}`.padEnd(40, 'g'), name: 'Hs7dKq2LmZ9xWv3RtYp1Qa', source: 'stem', groups: JSON.stringify([GROUP]), first_seen_at: now, last_seen_at: now, ...release })
.returning(['id', 'release_key'])
const [{ id }] = await FileStore.insert([
{ filename: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', poster: 'p@ngPost.com', date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: JSON.stringify([GROUP]),
segments: [{ number: 1, id: `${n.toString(16).padStart(32, 'a')}@ngPost`, size: 10 }], created_at: now, updated_at: now, ...file },
])
return { releaseId: Number(r.id), releaseKey: r.release_key as string, fileId: id }
}
test.group('Garbage filter (D42)', (group) => {
const explore = indexerConfig.garbage.explorePercent
group.each.setup(() => {
// Nothing explored, so a test's release is dropped whatever its key hashes to.
indexerConfig.garbage.explorePercent = 0
GarbageFilter.reset()
return testUtils.db().withGlobalTransaction()
})
group.each.teardown(() => {
indexerConfig.garbage.explorePercent = explore
GarbageFilter.reset()
})
group.teardown(() => QueueService.closeAll())
test('posting tools are told apart by their Message-IDs', ({ assert }) => {
assert.equal(uploadTool('FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'), 'nyuu')
assert.equal(uploadTool('b94268bc702040f7b811d5318f727e4f@ngPost'), 'ngPost')
assert.equal(uploadTool('part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local'), 'partNofM')
assert.equal(uploadTool('DLugnv3ZrVBv8Z9I69nU_19o112@JBinUp.local'), 'JBinUp')
assert.equal(uploadTool('t3Y4mV92YCp0McxDc5@zLZlsw.RYqes'), 'random')
assert.equal(uploadTool('seed-1-12@test'), 'other')
assert.equal(uploadTool(null), 'other')
})
test('the explored share is stable per key and about the percentage', ({ assert }) => {
assert.equal(explored('some-release', 2), explored('some-release', 2))
assert.isFalse(explored('some-release', 0))
const kept = Array.from({ length: 10_000 }, (_, i) => explored(`release-${i}`, 2)).filter(Boolean).length
assert.isAbove(kept, 150)
assert.isBelow(kept, 250)
})
test('seeding skips random-named files of a dropped signature, not readable or other ones', async ({ assert }) => {
await dropSignature('ngPost')
const aggregator = new FileAggregator<IngestSegment>()
const files = aggregator
.add([...upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2), ...upload('Some.Show.S01E01.1080p.WEB.part01.rar', 2), ...upload('Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar', 2, 'nyuu')])
.map(toSeedFile)
assert.lengthOf(files, 3)
const result = await SeedService.store(files)
assert.include(result, { files: 2, dropped: 1 })
assert.isNull(await db.from('files').where('filename', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
assert.isNull(await db.from('releases').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa').first())
assert.isNotNull(await db.from('files').where('filename', 'Some.Show.S01E01.1080p.WEB.part01.rar').first())
assert.isNotNull(await db.from('files').where('filename', 'Pq4ZmXc8VbN2LkJh6GfDs3.part01.rar').first())
})
test('ingest skips them too, and a release with a readable file is kept whole', async ({ assert }) => {
await dropSignature('ngPost')
await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2))
assert.isNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar').first())
// Off: everything is stored.
indexerConfig.garbage.enabled = false
try {
await IngestService.ingest(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar', 2))
} finally {
indexerConfig.garbage.enabled = true
}
assert.isNotNull(await db.from('binaries').where('name', 'Hs7dKq2LmZ9xWv3RtYp1Qa.part02.rar').first())
})
test('a release dropped after its probe is not stored again', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const files = aggregator.add(upload('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 2)).map(toSeedFile)
await db.table('dropped_releases').insert({ release_key: files[0].release!.key, reason: 'encrypted' })
assert.include(await SeedService.store(files), { files: 0, dropped: 1 })
})
test('after its probe, an encrypted random-named release without name or password is deleted', async ({ assert }) => {
const garbage = await storedRelease({ encrypted: true })
assert.equal(await GarbageFilter.afterProbe(garbage.fileId, 'rar5'), 'encrypted')
assert.isNull(await db.from('files').where('id', garbage.fileId).first())
assert.isNull(await db.from('releases').where('id', garbage.releaseId).first())
assert.equal((await db.from('dropped_releases').where('release_key', garbage.releaseKey).first()).reason, 'encrypted')
const container = await storedRelease({ container: 'rclone', encrypted: true })
assert.equal(await GarbageFilter.afterProbe(container.fileId, 'rclone'), 'rclone')
const missing = await storedRelease()
assert.equal(await GarbageFilter.afterProbe(missing.fileId, 'missing'), 'missing')
})
test('kept after the probe: a password, a real name, other probes waiting, nothing unreadable found', async ({ assert }) => {
const password = await storedRelease({ encrypted: true, password: 'secret', password_source: 'spot' })
assert.isNull(await GarbageFilter.afterProbe(password.fileId, 'rar5'))
const named = await storedRelease({ encrypted: true, name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
assert.isNull(await GarbageFilter.afterProbe(named.fileId, 'rar5'))
const waiting = await storedRelease({ encrypted: true })
const now = new Date()
await db.table('binaries').insert({ file_key: `waiting${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.par2', total_parts: 1, groups: '[]',
first_seen_at: now, last_seen_at: now, release_id: waiting.releaseId, awaiting_inspection: true })
assert.isNull(await GarbageFilter.afterProbe(waiting.fileId, 'rar5'))
const readable = await storedRelease()
assert.isNull(await GarbageFilter.afterProbe(readable.fileId, 'rar5'))
assert.isNotNull(await db.from('releases').where('id', readable.releaseId).first())
})
test('learns dropped signatures from probed releases, and keeps a verdict without enough probes', async ({ assert }) => {
const { minProbes } = indexerConfig.garbage
const now = new Date()
const releases = await db
.table('releases')
.insert(Array.from({ length: minProbes }, (_, i) => ({ release_key: `learn${i}`.padEnd(40, 'l'), name: `Hs7dKq2LmZ9xWv3RtYp${i}`, groups: '[]', first_seen_at: now, last_seen_at: now })))
.returning('id')
await db.table('signature_probes').insert(releases.map((r) => ({ release_id: r.id, tool: 'ngPost', group_name: GROUP })))
// Five probes of another signature: too few to decide.
await db.table('signature_probes').insert(releases.slice(0, 5).map((r, i) => ({ release_id: Number(r.id) + 1_000_000_000 + i, tool: 'nyuu', group_name: GROUP })))
const learned = await GarbageFilter.learn()
assert.include(learned.changed.join('\n'), `ngPost ${GROUP}: dropped`)
const rows = await db.from('upload_signatures').where('group_name', GROUP).orderBy('tool')
assert.deepEqual(rows.map((r) => [r.tool, r.dropped, r.probed]), [['ngPost', true, minProbes], ['nyuu', false, 5]])
// One real name in a hundred is above the 0.5% limit: kept again.
await db.from('releases').where('id', releases[0].id).update({ name: 'Some.Movie.2024.1080p.WEB-DL.x264-GRP' })
await GarbageFilter.learn()
assert.isFalse((await db.from('upload_signatures').where({ tool: 'ngPost', group_name: GROUP }).first()).dropped)
})
test('a stored file of a dropped signature is not probed, and stops waiting for it', async ({ assert }) => {
const stored = await storedRelease()
const now = new Date()
await db.table('binaries').insert({ file_key: `skip${n}`, poster: 'p', subject: 's', name: 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', total_parts: 1, groups: '[]',
first_seen_at: now, last_seen_at: now, release_id: stored.releaseId, file_id: stored.fileId, awaiting_inspection: true })
assert.isNotNull(await ContentService.prepare(stored.fileId))
await dropSignature('ngPost')
assert.isNull(await ContentService.prepare(stored.fileId))
assert.isNull((await db.from('binaries').where('file_id', stored.fileId).first()).awaiting_inspection)
})
})
tests/functional/ingest.spec.ts
-168
View File
@@ -1,168 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import QueueService from '#services/QueueService'
import { fileKey } from '#lib/subject/index'
import FileStore from '#services/FileStore'
function segment(overrides: Partial<IngestSegment>): IngestSegment {
const poster = 'ingest-test@example.invalid'
const subject = '"test-file.bin" yEnc'
return {
fileKey: fileKey(poster, subject, 3),
poster,
subject,
name: 'test-file.bin',
totalParts: 3,
number: 1,
messageId: 'part1@test',
bytes: 100,
groups: ['alt.binaries.test'],
...overrides,
}
}
async function binaryFor(key: string) {
return db.from('binaries').where('file_key', key).first()
}
// None of these cases complete a binary, so nothing is queued in Redis.
test.group('IngestService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
// IngestService opens the BullMQ queues; close them so the test process can exit.
group.teardown(() => QueueService.closeAll())
test('existing binaries are only rewritten when something changes', async ({ assert }) => {
const first = segment({ number: 1, postedAt: new Date('2026-09-28T12:00:00Z') })
await IngestService.ingest([first])
const before = await binaryFor(first.fileKey)
// A later part within TOUCH_MS: nothing to change, so the row is left alone.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', postedAt: new Date('2026-09-28T12:05:00Z') })])
const same = await binaryFor(first.fileKey)
assert.equal(+same.last_seen_at, +before.last_seen_at)
assert.equal(await db.from('parts').where('binary_id', before.id).count('* as n').then((r) => Number(r[0].n)), 2)
// An earlier post date and a stale last_seen_at are merged.
await db.from('binaries').where('id', before.id).update({ last_seen_at: new Date(Date.now() - 60 * 60 * 1000) })
await IngestService.ingest([segment({ number: 1, postedAt: new Date('2026-09-28T11:00:00Z') })])
const after = await binaryFor(first.fileKey)
assert.equal(after.posted_at.toISOString(), '2026-09-28T11:00:00.000Z')
assert.isAbove(+after.last_seen_at, Date.now() - 60_000)
})
test('a binary that is a file already takes no new header parts', async ({ assert }) => {
await IngestService.ingest([segment({ number: 1 })])
const binary = await binaryFor(segment({}).fileKey)
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'test-file.bin', poster: 'x', date: now.getTime(), parts: 3, size: 300, groups: '[]', segments: [], created_at: now, updated_at: now },
])
await db.from('binaries').where('id', binary.id).update({ file_id: file.id })
await db.from('parts').where('binary_id', binary.id).delete()
// The same articles again (a crosspost, a backfill range): nothing to store.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test' }), segment({ number: 3, messageId: 'part3@test' })])
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 0)
// Body-derived parts (with a per-part CRC) are still kept.
await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', pcrc32: 'abcd1234' })])
assert.equal(await db.from('parts').where('binary_id', binary.id).count('* as n').then((r) => Number(r[0].n)), 1)
})
test('ingesting the same segments twice is a no-op', async ({ assert }) => {
const segments = [segment({ number: 1 }), segment({ number: 2, messageId: 'part2@test' })]
await IngestService.ingest(segments)
await IngestService.ingest(segments)
const binary = await binaryFor(segments[0].fileKey)
const [{ count }] = await db.from('parts').where('binary_id', binary.id).count('* as count')
assert.equal(Number(count), 2)
assert.isNull(binary.completed_at)
assert.isNull(binary.conflicted_at)
})
test('a weak identity with two articles for one part number is flagged and never completes', async ({
assert,
}) => {
const weak = { weakIdentity: true }
await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'volume-a-1@test' })])
// Same name, poster and size, but another file: the second volume of a set.
await IngestService.ingest([
segment({ ...weak, number: 1, messageId: 'volume-b-1@test' }),
segment({ ...weak, number: 2, messageId: 'volume-b-2@test' }),
segment({ ...weak, number: 3, messageId: 'volume-b-3@test' }),
])
const binary = await binaryFor(segment({}).fileKey)
assert.isNotNull(binary.conflicted_at)
assert.isNull(binary.completed_at, 'all part numbers are present, but it must not complete')
})
test('a conflict inside a single batch is detected too', async ({ assert }) => {
await IngestService.ingest([
segment({ weakIdentity: true, number: 2, messageId: 'a@test' }),
segment({ weakIdentity: true, number: 2, messageId: 'b@test' }),
])
assert.isNotNull((await binaryFor(segment({}).fileKey)).conflicted_at)
})
test('on the subject path a second article for a part is a repost, not a conflict', async ({
assert,
}) => {
await IngestService.ingest([segment({ number: 1, messageId: 'original@test' })])
await IngestService.ingest([segment({ number: 1, messageId: 'repost@test' })])
const binary = await binaryFor(segment({}).fileKey)
assert.isNull(binary.conflicted_at)
const part = await db.from('parts').where({ binary_id: binary.id, number: 1 }).first()
assert.equal(part.message_id, 'original@test')
})
test('an obfuscated file joins a known par2 set of exactly its size', async ({ assert }) => {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'k'.repeat(40), name: 'Some.Show.S01E01', source: 'par2', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
await db.table('release_files').insert({ release_id: release.id, par2_file_id: 'f'.repeat(32), name: 'Some.Show.S01E01.mkv', size: 5467240532, md5_16k: '0'.repeat(32) })
const weak = { weakIdentity: true, postedAt: now, release: null }
await IngestService.ingest([
segment({ ...weak, fileKey: 'a'.repeat(40), size: 5467240532, totalParts: 7628, messageId: 'match@test' }),
segment({ ...weak, fileKey: 'b'.repeat(40), size: 5467240533, totalParts: 7628, messageId: 'other@test' }),
])
const matched = await binaryFor('a'.repeat(40))
const other = await binaryFor('b'.repeat(40))
assert.equal(Number(matched.release_id), Number(release.id))
assert.isNull(other.release_id)
})
test('an identical repost (same part CRC) is not a conflict', async ({ assert }) => {
const weak = { weakIdentity: true, pcrc32: 'abcd1234', partSize: 700 }
await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'first@test' })])
await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'repost@test' })])
const binary = await binaryFor(segment({}).fileKey)
assert.isNull(binary.conflicted_at)
await IngestService.ingest([segment({ ...weak, pcrc32: 'ffff0000', number: 1, messageId: 'other@test' })])
assert.isNotNull((await binaryFor(segment({}).fileKey)).conflicted_at)
})
test('a release takes its file total from whichever file carries the counter', async ({ assert }) => {
const { releaseIdentity } = await import('#lib/release/index')
const poster = 'archiver@example.invalid'
const file = (subject: string, name: string, key: string) =>
segment({ fileKey: key.padEnd(40, 'k'), poster, subject, name, totalParts: 1, messageId: `${key}@test`,
release: releaseIdentity({ poster, subject, name, weak: false }) })
// The counter-less par2 index comes first in the batch.
await IngestService.ingest([
file('"x.tar.zst.par2" yEnc 3100', 'x.tar.zst.par2', 'index'),
file('[1/2] - "x.tar.zst" yEnc 1000', 'x.tar.zst', 'archive'),
])
const release = await db.from('releases').where('name', 'x').where('poster', poster).first()
assert.equal(release.file_total, 2)
})
})
tests/functional/inspection_queue.spec.ts
-36
View File
@@ -1,36 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import QueueService from '#services/QueueService'
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
test.group('Inspection queue', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
test('the backlog stays in Postgres; BullMQ gets what fits, in priority order', async ({ assert }) => {
const queue = QueueService.collectionQueue
await queue.drain()
await InspectionQueue.add([9_000_001, 9_000_002], INSPECTION_PRIORITY.low)
await InspectionQueue.add([9_000_003])
await InspectionQueue.add([9_000_004], INSPECTION_PRIORITY.targeted)
// Queued again with a higher priority: one row, the higher priority.
await InspectionQueue.add([9_000_001], INSPECTION_PRIORITY.targeted)
assert.equal(await InspectionQueue.size(), 4)
const max = InspectionQueue.feedMax
;(InspectionQueue as any).feedMax = 2
try {
assert.equal(await InspectionQueue.feed(), 2)
const waiting = (await queue.getWaiting()).map((job) => job.data.fileId).sort()
assert.deepEqual(waiting, [9_000_001, 9_000_004])
// Full: nothing more until the workers take some.
assert.equal(await InspectionQueue.feed(), 0)
const left = (await db.from('inspection_queue').select('file_id')).map((r) => Number(r.file_id)).sort()
assert.deepEqual(left, [9_000_002, 9_000_003])
} finally {
;(InspectionQueue as any).feedMax = max
await queue.drain()
}
})
})
tests/functional/names.spec.ts
-94
View File
@@ -1,94 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import testUtils from '@adonisjs/core/services/test_utils'
import NameService from '#services/NameService'
async function release(name: string, files: { name: string; size: number; md5_16k: string }[]) {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: `${name}`.padEnd(40, 'x').slice(0, 40), name, source: 'par2', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
for (const [i, f] of files.entries()) {
await db.table('release_files').insert({ release_id: row.id, par2_file_id: String(i).padStart(32, '0'), name: f.name, size: f.size, md5_16k: f.md5_16k })
}
return Number(row.id)
}
async function binary(key: string, fields: Record<string, unknown>) {
const now = new Date()
const [row] = await db
.table('binaries')
.insert({ file_key: key.padEnd(40, '0'), poster: 'x', subject: '"x" yEnc', name: 'Xq8r2KdLm3Zw91PqA7', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, ...fields })
.returning('id')
return Number(row.id)
}
test.group('NameService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('an exact match leaves a random-named single-file release for the par2 set', async ({ assert }) => {
const season = await release('SEAL.Team.S03', [
{ name: 'SEAL.Team.S03E18.1080p.WEB-DL-nitrato.mkv', size: 939_684_767, md5_16k: 'd'.repeat(32) },
])
const now = new Date()
const [weak] = await db
.table('releases')
.insert({ release_key: 'w'.repeat(40), name: 'mOzyyxCyYUjAbCdEf12', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [file] = await FileStore.insert([
{ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', segments: [], release_id: weak.id, created_at: now, updated_at: now },
])
const episode = await binary('ep18', { size: 939_684_767, md5_16k: 'd'.repeat(32), release_id: weak.id, file_id: file.id })
await NameService.resolve([episode])
const moved = await db.from('binaries').where('id', episode).first()
assert.equal(Number(moved.release_id), season)
assert.equal(moved.real_name, 'SEAL.Team.S03E18.1080p.WEB-DL-nitrato.mkv')
assert.equal(Number((await db.from('files').where('id', file.id).first()).release_id), season)
})
test('a file in a real grouping keeps it', async ({ assert }) => {
await release('Other.Set', [{ name: 'Other.Show.S01E01.mkv', size: 5_000_000, md5_16k: 'e'.repeat(32) }])
const now = new Date()
const [nzb] = await db
.table('releases')
.insert({ release_key: 'n'.repeat(40), name: 'Qz8r2KdLm3Zw91PqA7x', source: 'nzb', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const id = await binary('nzbfile', { size: 5_000_000, md5_16k: 'e'.repeat(32), release_id: nzb.id })
await NameService.resolve([id])
assert.equal(Number((await db.from('binaries').where('id', id).first()).release_id), Number(nzb.id))
})
test('names equal-sized volumes exactly by their first-16k fingerprint', async ({ assert }) => {
const releaseId = await release('Some.Movie', [
{ name: 'Some.Movie.part1.rar', size: 50_000_000, md5_16k: 'a'.repeat(32) },
{ name: 'Some.Movie.part2.rar', size: 50_000_000, md5_16k: 'b'.repeat(32) },
])
const one = await binary('one', { size: 50_000_000, md5_16k: 'b'.repeat(32) })
const two = await binary('two', { size: 50_000_000, md5_16k: 'a'.repeat(32) })
const stranger = await binary('three', { size: 50_000_000, md5_16k: 'c'.repeat(32) })
await NameService.resolve([one, two, stranger])
const rows = await db.from('binaries').whereIn('id', [one, two, stranger]).orderBy('id')
assert.deepEqual(rows.map((r) => [r.real_name, r.release_id && Number(r.release_id)]), [
['Some.Movie.part2.rar', releaseId],
['Some.Movie.part1.rar', releaseId],
[null, null],
])
})
test('without a fingerprint, names by size only when the size is unique in the release', async ({ assert }) => {
const releaseId = await release('Other', [
{ name: 'Other.mkv', size: 700_000_000, md5_16k: 'd'.repeat(32) },
{ name: 'Other.part1.rar', size: 10_000_000, md5_16k: 'e'.repeat(32) },
{ name: 'Other.part2.rar', size: 10_000_000, md5_16k: 'f'.repeat(32) },
])
const video = await binary('video', { size: 700_000_000, release_id: releaseId })
const volume = await binary('volume', { size: 10_000_000, release_id: releaseId })
await NameService.resolve([video, volume])
assert.equal((await db.from('binaries').where('id', video).first()).real_name, 'Other.mkv')
assert.isNull((await db.from('binaries').where('id', volume).first()).real_name)
})
})
tests/functional/newznab.spec.ts
-91
View File
@@ -1,91 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import env from '#start/env'
import testUtils from '@adonisjs/core/services/test_utils'
let counter = 0
async function seed(name: string, opts: { fileTotal?: number; files?: number } = {}) {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: `nz${++counter}`.padEnd(40, 'n'), name, source: 'stem', poster: 'poster@example.invalid', groups: '["alt.binaries.test"]', file_total: opts.fileTotal ?? null, posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
for (let i = 0; i < (opts.files ?? 1); i++) {
await FileStore.insert([{
filename: `${name}.part${i + 1}.rar`, poster: 'poster@example.invalid', date: now.getTime(), parts: 1, size: 1_000_000,
release_id: release.id, groups: '["alt.binaries.test"]', segments: [{ number: 1, id: `${name}-${i}@test`, size: 1_000_000 }],
created_at: now, updated_at: now,
}])
}
return Number(release.id)
}
const ids = (xml: string) => [...xml.matchAll(/<guid isPermaLink="false">(\d+)<\/guid>/g)].map((m) => Number(m[1]))
test.group('Newznab API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('caps lists search functions and categories', async ({ client, assert }) => {
const response = await client.get('/api').qs({ t: 'caps' })
response.assertStatus(200)
assert.include(response.text(), '<tv-search available="yes" supportedParams="q,season,ep"/>')
assert.include(response.text(), '<category id="5000" name="TV">')
assert.include(response.text(), '<subcat id="5040" name="HD"/>')
})
test('tvsearch matches show, season and episode, with Newznab attributes', async ({ client, assert }) => {
const e02 = await seed('Nzbtest.Show.S01E02.1080p.WEB-DL-GRP')
const e03 = await seed('Nzbtest.Show.S01E03.1080p.WEB-DL-GRP')
const movie = await seed('Nzbtest.Show.The.Movie.2020.1080p.BluRay.x264-GRP')
const response = await client.get('/api').qs({ t: 'tvsearch', q: 'nzbtest show', season: '1', ep: '2' })
response.assertStatus(200)
const body = response.text()
assert.deepEqual(ids(body), [e02])
assert.include(body, '<newznab:attr name="category" value="5000"/>')
assert.include(body, '<newznab:attr name="category" value="5040"/>')
assert.include(body, '<newznab:attr name="season" value="S01"/>')
assert.include(body, `/api?t=get&amp;id=${e02}`)
assert.include(body, 'type="application/x-nzb"')
const season = await client.get('/api').qs({ t: 'tvsearch', q: 'nzbtest show', season: '1' })
assert.sameMembers(ids(season.text()), [e02, e03])
const movies = await client.get('/api').qs({ t: 'movie', q: 'nzbtest show' })
assert.deepEqual(ids(movies.text()), [movie])
})
test('leaves out obfuscated names and releases known to be incomplete', async ({ client, assert }) => {
const readable = await seed('Nzbtest.Other.2021.720p.WEB-DL-GRP')
const incomplete = await seed('Nzbtest.Partial.2021.720p.WEB-DL-GRP', { fileTotal: 5, files: 2 })
const random = await seed('Xq8r2KdLm3Zw91PqA7nzbtest')
const found = ids((await client.get('/api').qs({ t: 'search', q: 'nzbtest' })).text())
assert.include(found, readable)
assert.notInclude(found, incomplete)
assert.notInclude(found, random)
})
test('get returns the NZB, errors use Newznab codes', async ({ client, assert }) => {
const id = await seed('Nzbtest.Get.S02E01.720p.HDTV-GRP')
const nzb = await client.get('/api').qs({ t: 'get', id: String(id) })
nzb.assertStatus(200)
assert.include(nzb.header('content-type'), 'application/x-nzb')
assert.include(nzb.text(), 'Nzbtest.Get.S02E01.720p.HDTV-GRP-0@test')
assert.include((await client.get('/api').qs({ t: 'get', id: '999999999' })).text(), 'code="300"')
assert.include((await client.get('/api').qs({ t: 'nonsense' })).text(), 'code="202"')
})
test('requires the API key when one is configured', async ({ client, assert }) => {
const previous = env.get('NEWZNAB_API_KEY')
env.set('NEWZNAB_API_KEY', 'test-key' as any)
try {
assert.include((await client.get('/api').qs({ t: 'caps' })).text(), 'code="100"')
assert.include((await client.get('/api').qs({ t: 'caps', apikey: 'test-key' })).text(), '<caps>')
const id = await seed('Nzbtest.Key.S01E01.720p.HDTV-GRP')
const search = await client.get('/api').qs({ t: 'search', q: 'nzbtest key', apikey: 'test-key' })
assert.include(search.text(), `id=${id}&amp;apikey=test-key`)
} finally {
env.set('NEWZNAB_API_KEY', previous as any)
}
})
})
tests/functional/nzb_import.spec.ts
-137
View File
@@ -1,137 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import type { Segment } from '#lib/nzb/index'
import testUtils from '@adonisjs/core/services/test_utils'
import NzbImportService, { nzbFileName } from '#services/NzbImportService'
import SearchService from '#services/SearchService'
const XML = `<?xml version="1.0" encoding="utf-8"?>
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
<head><meta type="name">Real.Name.2026.1080p.WEB-DL-GRP</meta><meta type="password">hunter2</meta></head>
<file poster="p@x" date="1790598923" subject="Wond40 &#x22;Nachbe&#x22; Xq8r2KdLm3Zw91PqA7.part1.rar&#x22; yEnc (1/2)">
<groups><group>alt.binaries.bloaf</group></groups>
<segments><segment bytes="700" number="1">known-1@x</segment><segment bytes="700" number="2">known-2@x</segment></segments>
</file>
<file poster="p@x" date="1790598923" subject="&#x22;Xq8r2KdLm3Zw91PqA7.part2.rar&#x22; yEnc (1/1)">
<groups><group>alt.binaries.cores</group></groups>
<segments><segment bytes="500" number="1">new-1@x</segment></segments>
</file>
</nzb>`
async function file(fields: Record<string, unknown> & { segments: Segment[] }) {
const now = new Date()
const [{ id }] = await FileStore.insert([{ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }])
return db.from('files').where('id', id).first()
}
test.group('NzbImportService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('names entries despite odd quoting', ({ assert }) => {
assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar')
assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2')
// Spot NZBs without quotes.
assert.equal(nzbFileName('Adventure.part0055.rar - [056/175] (1/1)'), 'Adventure.part0055.rar')
assert.equal(nzbFileName('LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2 (1/0) (1/0)'), 'LegalPorno.26.03.01.XXX.1080p.MP4-P2P.vol007+008.par2')
assert.equal(nzbFileName('Reacher.S04.vol001+02.par2 yEnc'), 'Reacher.S04.vol001+02.par2')
assert.equal(nzbFileName('06 - A New Land, a New Music.flac Mehmet Ali Sanlikol - The Rise Up (2020) (01/35)'), '06 - A New Land, a New Music.flac')
assert.equal(nzbFileName('[01/10] - Some.Release.2026.part01.rar yEnc (1/50) 35000000'), 'Some.Release.2026.part01.rar')
assert.equal(nzbFileName('Mozart Zauberfloete 2026 mp4 720p.zip.001'), 'Mozart Zauberfloete 2026 mp4 720p.zip.001')
assert.equal(nzbFileName('LibreOffice_7.6.4_Win_x86-64.msi.vol0000'), 'LibreOffice_7.6.4_Win_x86-64.msi.vol0000')
})
test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', segments: [{ number: 1, id: 'nzb-1@x', size: 2000 }] })
const result = (await NzbImportService.import(posted, XML))!
assert.equal(result.releaseId, Number(release.id))
assert.include(result, { listed: 2, known: 1, added: 1, password: true })
const updated = await db.from('releases').where('id', release.id).first()
assert.equal(updated.name, 'Real.Name.2026.1080p.WEB-DL-GRP')
assert.equal(updated.password, 'hunter2')
assert.equal(updated.file_total, 2)
const files = await db.from('files').where('release_id', release.id).orderBy('filename').select('filename', 'groups')
assert.deepEqual(files.map((f) => f.filename), ['Xq8r2KdLm3Zw91PqA7.nzb', 'Xq8r2KdLm3Zw91PqA7.part1.rar', 'Xq8r2KdLm3Zw91PqA7.part2.rar'])
// Our own NZB passes the password on.
const nzb = await SearchService.nzb(Number(release.id))
assert.include(nzb!.xml, '<meta type="password">hunter2</meta>')
// Importing again changes nothing.
const again = (await NzbImportService.import(posted, XML))!
assert.equal(again.added, 0)
})
test('NZBs from one archive become separate releases named after their entries', async ({ assert }) => {
const nzbXml = (id: string) => `<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
<file poster="p@x" date="1790598923" subject="&#x22;${id}.flac&#x22; yEnc (1/1)">
<groups><group>alt.binaries.x</group></groups>
<segments><segment bytes="500" number="1">${id}@x</segment></segments>
</file></nzb>`
const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', segments: [{ number: 1, id: 'rar-1@x', size: 2000 }] })
const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))!
const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))!
assert.notEqual(a.releaseId, b.releaseId)
assert.equal(a.releaseName, 'The Kinks - Misfits-LP-1978')
assert.equal(b.releaseName, 'The Kinks - Sleepwalker-LP-1977')
const container = await db.from('files').where('id', archive.id).first()
assert.isNull(container.release_id)
})
test('a posted NZB alone in its own release moves to the release it describes', async ({ assert }) => {
const now = new Date()
const insert = async (key: string, name: string) =>
(await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id
const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT')
const lonely = await insert('lonely', '.nzb')
await file({ filename: 'x.part1.rar', release_id: target, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
const posted = await file({ filename: '.nzb', release_id: lonely, segments: [{ number: 1, id: 'nzb-2@x', size: 2000 }] })
await NzbImportService.import(posted, XML)
assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target))
assert.notExists(await db.from('releases').where('id', lonely).first())
})
test('a trusted NZB (a spot) takes its files into its own release, named after it', async ({ assert }) => {
const now = new Date()
const insertRelease = async (key: string, name: string, source: string) =>
(
await db
.table('releases')
.insert({ release_key: key.repeat(40).slice(0, 40), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
)[0].id
// Name rules had put the spot's first volume into a season release with another episode.
const season = await insertRelease('s', 'zxcvvbgnhgjytygf', 'stem')
await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: season, segments: [{ number: 1, id: 'known-1@x', size: 700 }] })
await file({ filename: 'other.episode.rar', release_id: season, segments: [{ number: 1, id: 'other-1@x', size: 700 }] })
const spotFile = { id: null, filename: 'x.nzb', poster: 'Bart', groups: [], date: now.getTime(), first_message_id: 'spot1@spot.net' }
const result = (await NzbImportService.import(spotFile, XML, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', moveContainer: false, trusted: true }))!
assert.notEqual(result.releaseId, Number(season))
const own = await db.from('releases').where('id', result.releaseId).first()
assert.include(own, { name: 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264', source: 'spot', file_count: 2 })
const left = await db.from('files').where('release_id', season).select('filename')
assert.deepEqual(left.map((f) => f.filename), ['other.episode.rar'])
// A par2 or archive name no longer replaces a spot's title.
const { isBetterName } = await import('#services/NameService')
assert.isTrue(isBetterName('zxcvvbgnhgjytygf', 'Better.Name.S03E285'))
// Another spot for the same files (a repost) joins the first one's release instead of taking them.
const repost = { ...spotFile, first_message_id: 'spot2@spot.net' }
const second = (await NzbImportService.import(repost, XML, { name: 'Repost', source: 'spot', moveContainer: false, trusted: true }))!
assert.equal(second.releaseId, result.releaseId)
assert.equal(second.releaseName, 'De.Bondgenoten.S03E285.DUTCH.1080p.WEB.H264')
})
})
tests/functional/par2_attach.spec.ts
-56
View File
@@ -1,56 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import Par2Service from '#services/Par2Service'
import type { Par2Set } from '#lib/par2/index'
const VOLUME = 10_000_952 // a standard split-volume size that recurs across posts
async function release(key: string, name: string) {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: key.padEnd(40, '0'), name, source: 'weak-stem', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
return Number(row.id)
}
async function binary(key: string, fields: Record<string, unknown>) {
const now = new Date()
const [row] = await db
.table('binaries')
.insert({ file_key: key.padEnd(40, '0'), poster: 'x', subject: '"x" yEnc', name: 'Xq8r2KdLm3Zw91PqA7', total_parts: 14, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now, ...fields })
.returning(['id', 'poster', 'groups', 'posted_at'])
return row
}
function set(id: string, files: { name: string; md5_16k: string }[]): Par2Set {
return {
recoverySetId: id.padEnd(32, '0'),
files: files.map((f, i) => ({ fileId: `${id}${i}`.padEnd(32, '0'), md5: '0'.repeat(32), md5_16k: f.md5_16k, length: VOLUME, name: f.name })),
}
}
test.group('Par2Service.attach', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a set does not join another post just because a size matches', async ({ assert }) => {
const other = await release('other', 'OtherPost')
await binary('other-vol', { size: VOLUME, md5_16k: 'a'.repeat(32), release_id: other })
const par2 = await binary('par2-b', { size: 50_000, total_parts: 1 })
const result = await Par2Service.attach(par2, set('b', [{ name: 'PostB.part1.rar', md5_16k: 'b'.repeat(32) }]))
assert.notEqual(result.releaseId, other)
assert.equal(await db.from('release_files').where('release_id', other).count('* as n').first().then((r) => Number(r.n)), 0)
})
test('an exactly matching file pulls the set into its release and gets its real name', async ({ assert }) => {
const stemRelease = await release('stem', 'Xq8r2KdLm3Zw91PqA7')
const vol = await binary('vol', { size: VOLUME, md5_16k: 'c'.repeat(32), release_id: stemRelease })
const par2 = await binary('par2-c', { size: 50_000, total_parts: 1 })
const result = await Par2Service.attach(par2, set('c', [{ name: 'Real.Name.part1.rar', md5_16k: 'c'.repeat(32) }]))
assert.equal(result.releaseId, stemRelease)
assert.equal((await db.from('binaries').where('id', vol.id).first()).real_name, 'Real.Name.part1.rar')
})
})
tests/functional/random_names.spec.ts
-27
View File
@@ -1,27 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import { looksRandom } from '#services/ArchiveService'
// Names seen on homelab (LOG.md 2026-10-01 (3)).
const RANDOM = [
'5d4063b423f57e1eadda1068b0091409', 'mOzyyxCyYUjAbCdEf12', '6327d9fa2d6acf3e9a61fb053359d5e7.1790574045',
'e63afdb4-6a1f-417d-aa1b-b14c0ea1b81d', '1650777167', '83111763 n', 'prvjr7hapd0i1ny14pz4', '8huowjcdamwbetufib9f',
'071vl0l120ndtjkjkuh34a2g6aqsak750f55dkudeaea8s4bivjm1l32fd3uduk0dab2t82hp553i29nfjvjt3redp', 'wHJ7s71sAGE', 'EmFqTfL8rxV',
'VIDEO_TS', 'Sample', 'CD1', 'Disc 2', 'Subs',
]
const READABLE = [
'Our.America.with.Lisa.Ling.2011.S01E01.Faith.Healers.1080p.DSCP.WEB-DL.DDP2.0.H.264-FUZEER', '00-graham_barham-club_country',
'1790274935_nicovideo_jp_watch_sm26602185', 'StarWars1977', 'nyuu_live_roundtrip', 'Makaja_Gonzales-Freestyle-WEB-2021-XTC_iNT',
'The Tom and Jerry Show (2014) S02E13 - Slap Happy Birthday', 'Bondgenoten', 'S01E02', '2026', 'Samples.Of.Love.2020.1080p', 'Subs.and.Dubs',
]
test.group('random names', () => {
test('looksRandom and SQL is_readable_name agree', async ({ assert }) => {
for (const name of [...RANDOM, ...READABLE]) {
const { rows } = await db.rawQuery('SELECT is_readable_name(?) AS readable', [name])
const random = RANDOM.includes(name)
assert.equal(looksRandom(name), random, `looksRandom(${name})`)
assert.equal(rows[0].readable, !random, `is_readable_name(${name})`)
}
})
})
tests/functional/release_complete.spec.ts
-94
View File
@@ -1,94 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
/** A release with `n` files (each with a binary of size 100000 + i), named f<i>.bin. */
async function release(n: number) {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: `complete-${n}`.padEnd(40, '0'), name: `Complete.Test.${n}`, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const releaseId = Number(row.id)
await db.rawQuery(
`WITH f AS (
INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
SELECT 'f' || i || '.bin', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'complete-' || ?::int || '-' || i, now(), now()
FROM generate_series(1, ?::int) i
RETURNING id, filename)
INSERT INTO binaries (file_key, poster, subject, name, total_parts, groups, first_seen_at, last_seen_at, size, file_id, release_id)
SELECT 'complete-b-' || ?::int || '-' || f.id, 'x', 'x', f.filename, 1, '[]', now(), now(), 100000 + substr(f.filename, 2, length(f.filename) - 5)::int, f.id, ?::bigint
FROM f`,
[releaseId, n, n, n, releaseId]
)
return releaseId
}
/** The par2 listing of `count` files f1..f<count> (sizes as in release()), in one INSERT, as Par2Service.attach does. */
async function listing(releaseId: number, count: number, offset = 0) {
await db.rawQuery(
`INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
SELECT ?::bigint, md5(i::text), 'f' || i || '.bin', 100000 + i, md5(i::text), 'set' FROM generate_series(1 + ?::int, ?::int + ?::int) i`,
[releaseId, offset, count, offset]
)
// Triggers only note the release; gc:scheduler's sweep recomputes it.
await ReleaseCompleteness.sweep()
return Boolean((await db.from('releases').where('id', releaseId).first()).complete)
}
test.group('release completeness', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('file inserts keep counts, size, extensions and search text current, once per statement', async ({ assert }) => {
const id = await release(3)
const row = await db.from('releases').where('id', id).first()
assert.equal(Number(row.file_count), 3)
assert.deepEqual(row.extensions, ['bin'])
// No par2 listing, no file total, not complete: its completeness can't change, so it isn't noted.
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
assert.equal(Number(count), 0)
})
test('a release with a par2 listing is noted once per insert, and complete by file names alone', async ({ assert }) => {
const now = new Date()
const [row] = await db
.table('releases')
.insert({ release_key: 'names-only'.padEnd(40, '0'), name: 'Names.Only', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const id = Number(row.id)
await db.rawQuery(
`INSERT INTO release_files (release_id, par2_file_id, name, size, md5_16k, set_id)
SELECT ?::bigint, md5(i::text), 'n' || i || '.rar', 100000 + i, md5(i::text), 'set' FROM generate_series(1, 3) i`,
[id]
)
await ReleaseCompleteness.sweep()
// Seeded files: no binaries, matched by name only.
await db.rawQuery(
`INSERT INTO files (filename, poster, date, parts, groups, segments, release_id, file_key, created_at, updated_at)
SELECT 'n' || i || '.rar', 'x', 0, 1, '[]', '\\x'::bytea, ?::bigint, 'names-only-' || i, now(), now() FROM generate_series(1, 3) i`,
[id]
)
const [{ count }] = await db.from('release_dirty').where('release_id', id).count('* as count')
assert.equal(Number(count), 1)
await ReleaseCompleteness.sweep()
assert.isTrue(Boolean((await db.from('releases').where('id', id).first()).complete))
})
test('complete when every listed file of 64 KiB or more is there, by size or name', async ({ assert }) => {
const id = await release(3)
assert.isTrue(await listing(id, 3))
const other = await release(2)
// f3 is listed but missing.
assert.isFalse(await listing(other, 3))
})
test('a par2 listing of thousands of files is checked in one go', async ({ assert }) => {
const id = await release(3000)
await ReleaseCompleteness.sweep()
const started = Date.now()
assert.isTrue(await listing(id, 3000))
// Was cubic: minutes for a few thousand rows.
assert.isBelow(Date.now() - started, 5000)
}).timeout(60_000)
})
tests/functional/release_merge.spec.ts
-42
View File
@@ -1,42 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import ReleaseMergeService from '#services/ReleaseMergeService'
let keys = 0
async function postRelease(name: string, files: string[], poster: string) {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: `merge${++keys}`.padEnd(40, 'm'), name, source: 'post', poster, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now })
.returning('id')
await FileStore.insert(
files.map((f, i) => ({ filename: f, poster, date: now.getTime(), parts: 1, size: 10, release_id: r.id, groups: '[]',
segments: [{ number: 1, id: `${name}-${poster}-${i}@x`, size: 10 }], created_at: now, updated_at: now }))
)
return Number(r.id)
}
test.group('Split uploads', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('merges one upload posted under a poster per volume, not reposts or other uploads', async ({ assert }) => {
const name = 'Masters.of.the.Universe.1987.Theatrical.Cut.2160p.BDRip-GRP'
const a = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part001.rar'], 'niSSYNy5zOW2VX@VMuLu_e7Gr.bIx')
const b = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part002.rar'], 'MgrZzFLB3D@MeK4_gntYXzh9g.LBQ')
const c = await postRelease(name, ['6c82910a43604dc5926ad25c45addcc5.part003.rar'], 'qNGVviOMN2AJrSzeXFZqvW@VzavS9GO.Y0b')
// A repost: the same file names again.
const repostName = 'Some.Repost.2026.1080p-GRP'
const r1 = await postRelease(repostName, ['abc.part01.rar'], 'p1@x')
const r2 = await postRelease(repostName, ['abc.part01.rar'], 'p2@x')
assert.deepEqual(await ReleaseMergeService.mergeSplitPosts({ dryRun: true }), { names: 1, releases: 2 })
await ReleaseMergeService.mergeSplitPosts()
assert.equal((await db.from('releases').where('id', a).first()).file_count, 3)
assert.equal((await db.from('releases').where('id', b).first()).file_count, 0)
assert.equal((await db.from('releases').where('id', c).first()).file_count, 0)
assert.equal((await db.from('releases').where('id', r1).first()).file_count, 1)
assert.equal((await db.from('releases').where('id', r2).first()).file_count, 1)
})
})
tests/functional/search_text.spec.ts
-41
View File
@@ -1,41 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import SearchService, { searchNormalize } from '#services/SearchService'
test.group('Search text', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('normalises like SQL does', async ({ assert }) => {
assert.equal(searchNormalize(' The.Block.AU_S22E35 - 1080p '), 'the block au s22e35 1080p')
const { rows } = await db.rawQuery(`SELECT search_normalize(' The.Block.AU_S22E35 - 1080p ') AS n`)
assert.equal(rows[0].n, 'the block au s22e35 1080p')
})
test('follows files and renames, only for visible releases', async ({ assert }) => {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: 's'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7Rt', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('*')
assert.isNull(r.search_text)
// A readable file name that the release name lacks joins the search text.
await FileStore.insert([
{ filename: 'Some.Documentary.2026.1080p.WEB.part01.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
groups: '[]', segments: [{ number: 1, id: 'st1@x', size: 10 }], created_at: now, updated_at: now },
{ filename: 'Q7wErTyUiOpAsDfGhJkL9.part02.rar', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: r.id,
groups: '[]', segments: [{ number: 1, id: 'st2@x', size: 10 }], created_at: now, updated_at: now },
])
let row = await db.from('releases').where('id', r.id).first()
assert.equal(row.search_text, 'xq8r2kdlm3zw91pqa7rt some documentary 2026 1080p web')
const found = await SearchService.search({ q: 'documentary 2026.1080p' } as any)
assert.isTrue(found.releases.some((x: any) => x.id === Number(r.id)))
// A rename recomputes it.
await db.from('releases').where('id', r.id).update({ name: 'Some.Documentary.2026.1080p.WEB-GRP' })
row = await db.from('releases').where('id', r.id).first()
assert.equal(row.search_text, 'some documentary 2026 1080p web grp some documentary 2026 1080p web')
})
})
tests/functional/seed.spec.ts
-87
View File
@@ -1,87 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import { segmentFromHeader, type IngestSegment } from '#services/IngestService'
import SeedService from '#services/SeedService'
import QueueService from '#services/QueueService'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile } from '#lib/seed/wire'
import { insertLimits, segmentsOf } from '#services/FileStore'
function header(subject: string, n: number) {
return {
'subject': subject,
'from': 'seeder <seed@example.invalid>',
'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
'message-id': `<seed-${n}-${subject.length}@test>`,
':bytes': '700000',
}
}
test.group('SeedService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(() => QueueService.closeAll())
test('stores aggregated files with their release and a completed binary', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const subjects = [1, 2].map((i) => `Seed.Test.S01E01.1080p - [1/2] - "Seed.Test.S01E01.1080p.part${i}.mkv" yEnc (%d/3)`)
const segments = subjects.flatMap((s) => [1, 2, 3].map((n) => segmentFromHeader(header(s.replace('%d', String(n)), n), 'alt.binaries.test', n)!))
const files = aggregator.add(segments).map(toSeedFile)
assert.lengthOf(files, 2)
const result = await SeedService.store(files)
assert.include(result, { files: 2, existing: 0, releases: 1 })
const stored = await db.from('files').where('filename', 'Seed.Test.S01E01.1080p.part1.mkv').first()
assert.equal(stored.parts, 3)
assert.deepEqual(segmentsOf(stored).map((s) => s.number), [1, 2, 3])
const release = await db.from('releases').where('id', stored.release_id).first()
assert.equal(release.name, segments[0].release!.name) // named as live indexing would
// A readable name isn't inspected: no binary (LOG.md 2026-10-06 (11)).
assert.isNull(await db.from('binaries').where('file_key', stored.file_key).first())
// The same files again (a crosspost, a second run): nothing new.
const again = await SeedService.store(files)
assert.include(again, { files: 0, existing: 2 })
})
test('an obfuscated file keeps a binary until inspection has fingerprinted it', async ({ assert }) => {
const aggregator = new FileAggregator<IngestSegment>()
const subject = 'Xq8r2KdLm3Zw91PqA7zz [1/1] - "Xq8r2KdLm3Zw91PqA7zz.bin" yEnc (%d/2)'
const segments = [1, 2].map((n) => segmentFromHeader(header(subject.replace('%d', String(n)), n), 'alt.binaries.test', n)!)
const files = aggregator.add(segments).map(toSeedFile)
const result = await SeedService.store(files)
assert.include(result, { files: 1, inspections: 1 })
const binary = await db.from('binaries').where('file_key', files[0].fileKey).first()
assert.isTrue(binary.awaiting_inspection)
assert.isNotNull(binary.file_id)
})
test('stores a batch in several INSERTs', async ({ assert }) => {
// Postgres takes 65,535 bind parameters per statement: ~5,400 rows of 12 columns.
const rows = insertLimits.rows
insertLimits.rows = 50
try {
const aggregator = new FileAggregator<IngestSegment>()
const segments = Array.from({ length: 120 }, (_, i) =>
segmentFromHeader(header(`Dense.Upload - [${i + 1}/120] - "dense.${i + 1}.dat" yEnc (1/1)`, i + 1), 'alt.binaries.test', i + 1)!
)
const files = aggregator.add(segments).map(toSeedFile)
assert.include(await SeedService.store(files), { files: 120 })
const [{ count }] = await db.from('files').whereLike('filename', 'dense.%').count('* as count')
assert.equal(Number(count), 120)
} finally {
insertLimits.rows = rows
}
})
test('fragments are kept near the range edges only', async ({ assert }) => {
const subject = (name: string) => `Seed.Frag - "${name}" yEnc (1/3)`
const edge = segmentFromHeader(header(subject('edge.bin'), 1), 'alt.binaries.test', 1_000_100)!
const middle = segmentFromHeader(header(subject('middle.bin'), 2), 'alt.binaries.test', 1_500_000)!
const kept = await SeedService.storeFragments([toFragment(edge), toFragment(middle)], { start: 1_000_000, end: 2_999_999 })
assert.equal(kept, 1)
assert.isOk(await db.from('binaries').where('name', 'edge.bin').first())
assert.isNotOk(await db.from('binaries').where('name', 'middle.bin').first())
})
})
tests/functional/seed_api.spec.ts
-265
View File
@@ -1,265 +0,0 @@
import { test } from '@japa/runner'
import { gzipSync } from 'node:zlib'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import SeedOrchestrator from '#services/SeedOrchestrator'
import QueueService from '#services/QueueService'
import indexerConfig from '#config/indexer'
import SeedFetchPool from '#services/SeedFetchPool'
import { deflateSync } from 'node:zlib'
import { segmentFromHeader } from '#lib/ingest/segment'
import { FileAggregator } from '#lib/aggregate/index'
import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
import type { IngestSegment } from '#lib/ingest/segment'
import FileStore from '#services/FileStore'
import RemoteContentService from '#services/RemoteContentService'
import { Job } from 'bullmq'
function header(subject: string, n: number) {
return { 'subject': subject, 'from': 'api seeder <api@example.invalid>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<api-${n}-${subject.length}@test>`, ':bytes': '500000' }
}
function batch(next: number): SeedBatch {
const segments = [1, 2].map((n) => segmentFromHeader(header(`Api.Test.2026 - "Api.Test.2026.mkv" yEnc (${n}/2)`, n), 'alt.binaries.test', n)!)
const lone = segmentFromHeader(header('Api.Test.2026 - "Api.Test.2026.nfo" yEnc (1/3)', 9), 'alt.binaries.test', 9)!
const aggregator = new FileAggregator<IngestSegment>()
return { next, headers: 3, obfuscated: 0, deferred: 0, files: aggregator.add(segments).map(toSeedFile), fragments: [toFragment(lone)] }
}
test.group('Seed API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.teardown(async () => {
await RemoteContentService.close()
await QueueService.closeAll()
})
test('a node leases a range, posts a gzip batch, and finishes it', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('test-node')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
;(await client.post('/api/v1/seed/lease')).assertStatus(401)
const leased = await client.post('/api/v1/seed/lease').headers(auth)
leased.assertStatus(200)
const range = leased.body().range
assert.include(range, { group: 'alt.binaries.test', start: 1, end: 100 })
// As seed:node sends it: a gzip-compressed JSON body.
const posted = await fetch(`http://${process.env.HOST}:${process.env.PORT}/api/v1/seed/ranges/${range.id}/batches`, {
method: 'POST',
headers: { ...auth, 'content-type': 'application/json', 'content-encoding': 'gzip' },
body: gzipSync(JSON.stringify(batch(51))),
})
assert.equal(posted.status, 200)
assert.include(((await posted.json()) as any).stored, { files: 1 })
const file = await db.from('files').where('filename', 'Api.Test.2026.mkv').first()
assert.equal(file.parts, 2)
// The lone .nfo segment sits near the range's edge: kept as a binary (not staged) for the neighbouring range.
assert.isOk(await db.from('binaries').where('name', 'Api.Test.2026.nfo').first())
const row = await db.from('seed_ranges').where('id', range.id).first()
assert.equal(Number(row.next_id), 51)
assert.equal(Number(row.headers), 3)
;(await client.post(`/api/v1/seed/ranges/${range.id}/done`).headers(auth)).assertStatus(200)
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done')
;(await client.post('/api/v1/seed/lease').headers(auth)).assertStatus(204)
})
test('a quiet lease passes to another node, and the first one gets 409', async ({ client, assert }) => {
const first = await SeedOrchestrator.issueToken('node-a')
const second = await SeedOrchestrator.issueToken('node-b')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const range = (await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${first}` })).body().range
await db.from('seed_ranges').where('id', range.id).update({ next_id: 40, heartbeat_at: new Date(Date.now() - 60 * 60 * 1000) })
const taken = (await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${second}` })).body().range
assert.equal(taken.id, range.id)
assert.equal(taken.start, 40)
const late = await client.post(`/api/v1/seed/ranges/${range.id}/batches`).headers({ authorization: `Bearer ${first}` }).json(batch(60))
late.assertStatus(409)
;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers({ authorization: `Bearer ${second}` }).json({ error: 'test' })).assertStatus(200)
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'queued')
})
test('no leases while the database is over the disk limit', async ({ client }) => {
const token = await SeedOrchestrator.issueToken('full-disk')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const limit = indexerConfig.backfill.maxDefaultGb
indexerConfig.backfill.maxDefaultGb = 0
try {
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204)
} finally {
indexerConfig.backfill.maxDefaultGb = limit
}
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(200)
})
test('a range given back before it started does not use up an attempt', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('no-news-server')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range
;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers(auth).json({ error: 'NNTP login: 502', uncounted: true })).assertStatus(200)
const row = await db.from('seed_ranges').where('id', range.id).first()
assert.equal(row.status, 'queued')
assert.equal(row.attempts, 0)
})
test('relay: a leased range gets its overview chunks through the API', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('relay-node')
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
const auth = { authorization: `Bearer ${token}` }
const range = (await client.post('/api/v1/seed/lease').headers(auth)).body().range
const fetch = SeedFetchPool.fetch
const connections = indexerConfig.seed.nntpConnections
const asked: unknown[] = []
SeedFetchPool.fetch = async (group, from, to) => {
asked.push([group, from, to])
return deflateSync(Buffer.from('5\tsubject\r\n'))
}
try {
indexerConfig.seed.nntpConnections = 0
;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 1, to: 50 }).headers(auth)).assertStatus(503)
indexerConfig.seed.nntpConnections = 1
const chunk = await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 50, to: 500 }).headers(auth)
chunk.assertStatus(200)
// Clamped to the range.
assert.deepEqual(asked, [['alt.binaries.test', 50, 100]])
;(await client.get(`/api/v1/seed/ranges/${range.id}/chunk`).qs({ from: 200, to: 300 }).headers(auth)).assertStatus(400)
} finally {
SeedFetchPool.fetch = fetch
indexerConfig.seed.nntpConnections = connections
}
})
test('ranges of a higher priority are leased first', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('priority-node')
await db.table('seed_ranges').insert([
{ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
{ group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101, priority: 1 },
])
const leased = await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })
assert.include(leased.body().range, { start: 101, end: 200 })
})
test('priorities take turns: half the leases go to the higher priority', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('turns-node')
const auth = { authorization: `Bearer ${token}` }
await db.table('seed_ranges').insert([
{ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 },
{ group_name: 'alt.binaries.test', start_id: 101, end_id: 200, next_id: 101 },
{ group_name: 'alt.binaries.test', start_id: 1001, end_id: 1100, next_id: 1001, priority: 1 },
{ group_name: 'alt.binaries.test', start_id: 1101, end_id: 1200, next_id: 1101, priority: 1 },
])
const starts: number[] = []
for (let i = 0; i < 4; i++) starts.push((await client.post('/api/v1/seed/lease').headers(auth)).body().range.start)
assert.deepEqual(starts, [1001, 1, 1101, 101])
})
test('a node inspects a file: lease, probe result, applied here (D41)', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'i'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7zz', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [file] = await FileStore.insert([
{
filename: 'Xq8r2KdLm3Zw91PqA7zz.part01.rar',
file_key: 'inspect-node-file'.padEnd(40, '0'),
poster: 'x',
date: now.getTime(),
parts: 2,
groups: '[]',
segments: [
{ number: 1, id: 'one@test', size: 100 },
{ number: 2, id: 'two@test', size: 100 },
],
release_id: release.id,
created_at: now,
updated_at: now,
},
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const leased = await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)
leased.assertStatus(200)
const job = leased.body().jobs.find((j: any) => j.task.kind === 'inspect' && j.task.fileId === file.id)
assert.deepEqual(job.task.segments.map((s: any) => s.id), ['one@test', 'two@test'])
// Locked: nobody else gets it meanwhile.
;(await client.post('/api/v1/seed/inspect/lease').json({ max: 5 }).headers(auth)).assertStatus(204)
// What probeInspection found in the first segment; posted as a JSON string, empty strings and all.
const result = {
size: 15_728_640,
contentType: null,
md5_16k: null,
yencName: '',
archive: {
format: 'rar5', volume: true, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, next: 120,
entries: [{ name: 'Real.Release.Name.2026.1080p-GRP/real.release.name.mkv', size: 5_000_000_000, packedSize: 15_700_000, directory: false, encrypted: true, splitBefore: false, splitAfter: true, stored: false, dataOffset: 100 }],
},
}
const done = { queue: job.queue, jobId: job.jobId, token: job.token, result: JSON.stringify(result) }
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(200)
const row = await db.from('releases').where('id', release.id).first()
assert.equal(row.name, 'Real.Release.Name.2026.1080p-GRP')
assert.isTrue(row.encrypted)
assert.lengthOf(row.contents, 1)
assert.isOk((await db.from('file_details').where('file_id', file.id).first())?.archive)
assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
// Too late (or twice): the lock is gone.
;(await client.post('/api/v1/seed/inspect/done').json(done).headers(auth)).assertStatus(409)
})
test('a node gives an inspection job back, or reports it failed', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node-2')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'Zk3m9QpL2xW8rT5vN1bY.bin', file_key: 'inspect-node-file-2'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'x@test', size: 1 }], created_at: now, updated_at: now },
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const lease = async () => (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
const first = await lease()
;(await client.post('/api/v1/seed/inspect/release').json({ queue: first.queue, jobId: first.jobId, token: first.token }).headers(auth)).assertStatus(200)
const again = await lease()
assert.equal(again.jobId, first.jobId)
;(await client.post('/api/v1/seed/inspect/failed').json({ queue: again.queue, jobId: again.jobId, token: again.token, error: 'NNTP BODY: no answer within 120 s' }).headers(auth)).assertStatus(200)
// Retried later (backoff), like a local failure.
assert.include(['delayed', 'waiting'], await (await Job.fromId(QueueService.collectionQueue, again.jobId))!.getState())
})
test('a missing first article reported by a node is recorded, not retried', async ({ client, assert }) => {
await QueueService.collectionQueue.drain()
await QueueService.archiveWalkQueue.drain()
const token = await SeedOrchestrator.issueToken('inspect-node-3')
const auth = { authorization: `Bearer ${token}` }
const now = new Date()
const [file] = await FileStore.insert([
{ filename: 'Wq7n3LpX9vT2rK8mZ5cYbA.bin', file_key: 'inspect-node-file-3'.padEnd(40, '0'), poster: 'x', date: now.getTime(), parts: 1, groups: '[]', segments: [{ number: 1, id: 'gone@test', size: 1 }], created_at: now, updated_at: now },
])
await QueueService.collectionQueue.add('process-collection', { fileId: file.id }, { jobId: `node-test-${file.id}-${Date.now()}` })
const job = (await client.post('/api/v1/seed/inspect/lease').json({ max: 1 }).headers(auth)).body().jobs[0]
const error = 'Seed API /api/v1/seed/articles?id=gone%40test: 410 {"error":"No such article"}'
;(await client.post('/api/v1/seed/inspect/failed').json({ queue: job.queue, jobId: job.jobId, token: job.token, error }).headers(auth)).assertStatus(200)
assert.equal((await db.from('file_details').where('file_id', file.id).first())?.content_type, 'missing')
assert.equal(await (await Job.fromId(QueueService.collectionQueue, job.jobId))!.getState(), 'completed')
})
test('a range reported to its end but never finished is marked done, not leased', async ({ client, assert }) => {
const token = await SeedOrchestrator.issueToken('late-node')
const [range] = await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 101 }).returning('id')
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204)
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'done')
})
})
tests/functional/sfv.spec.ts
-59
View File
@@ -1,59 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import SfvService from '#services/SfvService'
import { parseSfv } from '#lib/content/index'
const POSTER = 'Snazzy Poster <me@localhost.localdomain>'
let keys = 0
async function release(name: string, source = 'stem') {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: `sfv${++keys}`.padEnd(40, 'x'), name, source, groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
return Number(r.id)
}
async function file(name: string, releaseId: number, poster = POSTER) {
const [{ id }] = await FileStore.insert([
{ filename: name, poster, date: Date.now(), parts: 1, size: 10, release_id: releaseId, groups: '[]',
segments: [{ number: 1, id: `${name}@x`, size: 10 }], created_at: new Date(), updated_at: new Date() },
])
return db.from('files').where('id', id).first()
}
test.group('SFV grouping', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('parses SFV listings', ({ assert }) => {
assert.deepEqual(parseSfv('; made by x\r\n01-a-b-4066667a.flac 4066667A\r\n"02 c.flac" 5058F120\r\n\r\n'), [
{ name: '01-a-b-4066667a.flac', crc: '4066667a' },
{ name: '02 c.flac', crc: '5058f120' },
])
})
test("gathers an album's tracks into the SFV's release, named after the album; later tracks join too", async ({ assert }) => {
const album = await release('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026')
const sfv = await file('00-graham_barham-club_country_(after_party)-16bit-web-flac-2026.sfv', album)
const track1 = await release('01-graham_barham-country_music-4066667a')
await file('01-graham_barham-country_music-4066667a.flac', track1)
// Another poster's file with the same name stays.
const other = await release('01-graham_barham-country_music-4066667a', 'stem')
const foreign = await file('01-graham_barham-country_music-4066667a.flac', other, 'someone@else')
const listing = '01-graham_barham-country_music-4066667a.flac 4066667a\n02-graham_barham-sippin-5058f120.flac 5058f120\n'
assert.equal(await SfvService.attach(sfv, listing), 1)
const renamed = await db.from('releases').where('id', album).first()
assert.equal(renamed.name, 'graham_barham-club_country_(after_party)-16bit-web-flac-2026')
assert.equal((await db.from('files').where('id', foreign.id).first()).release_id, other)
// Track 2 completes after the SFV was read.
const track2 = await release('02-graham_barham-sippin-5058f120')
const late = await file('02-graham_barham-sippin-5058f120.flac', track2)
assert.equal(await SfvService.claim({ id: late.id, name: late.filename, poster: late.poster, date: Number(late.date), release_id: track2 }), album)
assert.equal((await db.from('releases').where('id', album).first()).file_count, 3)
})
})
tests/functional/split.spec.ts
-84
View File
@@ -1,84 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import IngestService, { type IngestSegment } from '#services/IngestService'
import SplitService from '#services/SplitService'
import { weakFileKey } from '#lib/subject/index'
import { crc32Hex } from '#lib/yenc/index'
// Three 4-byte parts per volume, with real CRCs so the split volumes verify.
const DATA: Record<string, Buffer> = { A: Buffer.from('AAAAaaaa1111'), B: Buffer.from('BBBBbbbb2222') }
const KEY = weakFileKey('a8f3Kq', 10_000_952, 3)
function article(volume: string, number: number, articleNumber: number): IngestSegment {
return {
fileKey: KEY,
poster: `random-${articleNumber}@example.invalid`,
subject: '"a8f3Kq" yEnc',
name: `random${articleNumber}`,
totalParts: 3,
number,
messageId: `${volume}-${number}@test`,
bytes: 700_000,
articleNumber,
groups: ['alt.binaries.test'],
weakIdentity: true,
size: 10_000_952,
release: null,
pcrc32: crc32Hex(DATA[volume].subarray((number - 1) * 4, number * 4)),
partSize: 4,
fileCrc32: number === 3 ? crc32Hex(DATA[volume]) : undefined,
}
}
test.group('SplitService', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
group.each.setup(() => {
// Keep completed volumes out of the real file queue.
const enqueue = IngestService.enqueue
IngestService.enqueue = async () => {}
return () => {
IngestService.enqueue = enqueue
}
})
test('splits two equal-sized volumes into verified, complete files', async ({ assert }) => {
// Volume A then volume B, arriving interleaved and out of order.
await IngestService.ingest([article('A', 1, 101), article('A', 2, 102), article('B', 1, 104)])
await IngestService.ingest([article('B', 2, 105), article('A', 3, 103), article('B', 3, 106)])
const parent = await db.from('binaries').where('file_key', KEY).first()
assert.isNotNull(parent.conflicted_at)
const result = await SplitService.splitPending()
assert.equal(result.volumes, 2)
assert.equal(result.completed, 2)
const volumes = await db.from('binaries').where('parent_binary_id', parent.id).orderBy('id')
assert.lengthOf(volumes, 2)
for (const [i, volume] of volumes.entries()) {
assert.isNotNull(volume.completed_at)
const parts = await db.from('parts').where('binary_id', volume.id).orderBy('number').select('message_id')
const letter = i === 0 ? 'A' : 'B'
assert.deepEqual(parts.map((p) => p.message_id), [1, 2, 3].map((n) => `${letter}-${n}@test`))
}
// Nothing new since the split: the next run leaves it alone.
assert.equal((await SplitService.splitPending()).parents, 0)
})
test('a mix that fails CRC verification is not turned into a file', async ({ assert }) => {
// B's last part is missing: the only complete candidate would mix volumes.
await IngestService.ingest([article('A', 1, 101), article('B', 1, 102), article('A', 2, 103)])
await IngestService.ingest([article('B', 2, 104), article('A', 3, 105)])
const parent = await db.from('binaries').where('file_key', KEY).first()
const result = await SplitService.splitPending()
const volumes = await db.from('binaries').where('parent_binary_id', parent.id)
for (const volume of volumes) {
const parts = await db.from('parts').where('binary_id', volume.id).select('message_id')
assert.lengthOf(new Set(parts.map((p) => p.message_id[0])), 1)
}
assert.isAtMost(result.completed, 1)
})
})
tests/functional/spot_listing.spec.ts
-193
View File
@@ -1,193 +0,0 @@
import { test } from '@japa/runner'
import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import SpotService, { isDummyNzb, NzbUnreadable } from '#services/SpotService'
import SearchService from '#services/SearchService'
test.group('SpotService listings', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('old verified spots become visible releases without files', async ({ assert }) => {
const now = new Date()
const old = new Date('2019-05-01T12:00:00Z')
const spot = {
message_id: 'old1@spot.net', group_name: 'free.pt', article_number: 42, title: 'Some.Show.S01E01.1080p.WEB-DL-GRP',
poster: 'Bart', key_id: 7, size: 1_500_000_000, posted_at: old, verified: true, created_at: now, status: 'pending',
}
await db.table('spots').insert(spot)
const [claimed] = await SpotService.claim(10)
assert.isTrue(SpotService.isListable(claimed))
assert.isFalse(SpotService.isListable({ ...claimed, title: 'DISPOSE abc@spot.net', key_id: 2 }))
assert.isFalse(SpotService.isListable({ ...claimed, verified: false }))
assert.equal(await SpotService.list([claimed]), 1)
const row = await db.from('spots').where('message_id', 'old1@spot.net').first()
assert.equal(row.status, 'listed')
const release = await db.from('releases').where('id', row.release_id).first()
assert.include(release, { name: 'Some.Show.S01E01.1080p.WEB-DL-GRP', source: 'spot', file_count: 0 })
assert.equal(Number(release.size), 1_500_000_000)
// The key an import of this spot's NZB uses, so it fills this release.
assert.equal(release.release_key, createHash('sha1').update('nzb\nold1@spot.net').digest('hex'))
const found = await SearchService.search({ q: 'Some Show S01E01' } as any)
assert.isTrue(found.releases.some((r: any) => r.id === Number(release.id) && r.listingOnly))
})
test('records a full XOVER batch of spots (more rows than one INSERT can bind)', async ({ assert }) => {
// A real spot's From line; the message ids differ, so these don't verify, which doesn't matter here.
const from = 'Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9>'
const overviews = Array.from({ length: 6000 }, (_, i) => [
i + 1,
{ 'from': from, 'subject': `Title ${i} | Bart`, 'message-id': `<bulk${i}@spot.net>`, 'date': 'Mon, 28 Sep 2026 18:24:45 GMT' },
]) as any
assert.equal(await SpotService.recordHeaders('free.pt', overviews), 6000)
const [{ count }] = await db.from('spots').where('message_id', 'like', 'bulk%@spot.net').count('* as count')
assert.equal(Number(count), 6000)
})
test('a release shows its spot: description, image and Spotweb categories', async ({ assert }) => {
const spot = {
message_id: 'details1@spot.net', group_name: 'free.pt', article_number: 43, title: 'Some.Movie.1981.1080p.BluRay-GRP',
poster: 'Bart', key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true,
created_at: new Date(), status: 'pending', category: 0, subcats: JSON.stringify({ a: [9], z: [0] }),
description: 'Plot[br]more', image: JSON.stringify({ segments: ['img@spot.net'], width: 185, height: 278 }),
details_at: new Date(),
}
await db.table('spots').insert(spot)
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
const { release_id: releaseId } = await db.from('spots').where('message_id', 'details1@spot.net').first()
const release = (await SearchService.release(Number(releaseId)))!
assert.equal(release.spot!.description, 'Plot[br]more')
assert.deepEqual(release.spot!.image, { url: `/api/v1/releases/${releaseId}/image`, width: 185, height: 278 })
assert.equal(release.spot!.categories.head, 'Image')
assert.deepEqual(release.spot!.categories.subcats, [{ type: 'Type', values: ['Movie'] }, { type: 'Format', values: ['x264'] }])
})
test('recognises placeholder NZBs (Dummy.rar) by their size against the spot', ({ assert }) => {
const nzb = (bytes: number[]) => ({ files: bytes.map((b) => ({ segments: [{ bytes: b }] })) })
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 45_000_000_000))
assert.isFalse(isDummyNzb(nzb([700_000_000, 700_000_000]), 1_500_000_000))
// Text-only spots announce the dummy's own size.
assert.isTrue(isDummyNzb(nzb([1113, 1506]), 2619))
})
})
test.group('SpotService passwords', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a password in the description goes on the release, and shows as known', async ({ assert }) => {
const now = new Date()
await db.table('spots').insert({
message_id: 'pw1@spot.net', group_name: 'free.pt', article_number: 44, title: 'Some.Film.2020.1080p-GRP', poster: 'Bart',
key_id: 7, size: 1_000_000, posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: now, status: 'pending',
description: 'Mooie film.[br][br]WW: p3IU5Rz9R', details_at: now,
})
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
assert.equal(await SpotService.applyPassword('pw1@spot.net'), 'p3IU5Rz9R')
const { release_id: releaseId } = await db.from('spots').where('message_id', 'pw1@spot.net').first()
await db.from('releases').where('id', releaseId).update({ encrypted: true })
const release = (await SearchService.release(Number(releaseId)))!
assert.include(release, { password: 'p3IU5Rz9R', passwordSource: 'spot', passwordState: 'known' })
// An NZB's own password is not replaced.
await db.from('releases').where('id', releaseId).update({ password: 'from-nzb', password_source: 'nzb' })
await SpotService.applyPassword('pw1@spot.net')
assert.equal((await db.from('releases').where('id', releaseId).first()).password, 'from-nzb')
})
test('password states', async ({ assert }) => {
const { passwordState } = await import('#services/SearchService')
assert.equal(passwordState(true, false), 'missing')
assert.equal(passwordState(true, true), 'known')
assert.equal(passwordState(null, true), 'known')
assert.equal(passwordState(false, false), 'none')
assert.equal(passwordState(null, false), 'none')
})
})
test.group('SpotService reading listings', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
const nzb = (password: string | null, bytes = 700_000_000) => `<?xml version="1.0" encoding="UTF-8"?>
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb"><head>${password ? `<meta type="password">${password}</meta>` : ''}</head>
<file poster="p" date="1700000000" subject="&quot;a.part1.rar&quot; yEnc (1/1)"><groups><group>alt.binaries.test</group></groups>
<segments><segment bytes="${bytes}" number="1">a1@x</segment></segments></file></nzb>`
/** A listed spot, as worker:spots leaves an old one, with its article and NZB as given. */
async function listedSpot(id: string, read: { description?: string; nzb?: string | null; error?: string; nzbError?: Error }) {
await db.table('spots').insert({
message_id: id, group_name: 'free.pt', article_number: 1, title: `Listed ${id}`, poster: 'Bart', key_id: 7, size: 700_000_000,
posted_at: new Date('2019-05-01T12:00:00Z'), verified: true, created_at: new Date(), status: 'pending',
})
const [claimed] = await SpotService.claim(10)
await SpotService.list([claimed])
const service = SpotService as any
service.readHead = async () => {
if (read.error) throw new Error(read.error)
return { header: { size: 700_000_000 }, xml: { description: read.description ?? '', nzbSegments: ['n@x'] } }
}
service.readNzb = async () => {
if (read.nzbError) throw read.nzbError
return read.nzb ?? null
}
}
const restore = () => {
delete (SpotService as any).readHead
delete (SpotService as any).readNzb
}
group.each.teardown(restore)
const passwordOf = async (id: string) => {
const spot = await db.from('spots').where('message_id', id).first()
const release = await db.from('releases').where('id', spot.release_id).first()
return { spot, password: release.password, source: release.password_source }
}
test("reads the NZB's password and the details, without importing files", async ({ assert }) => {
await listedSpot('l1@spot.net', { description: 'Plot', nzb: nzb('fromnzb1') })
assert.deepEqual(await SpotService.readListings(10, 2), { 'nzb-password': 1 })
const { spot, password, source } = await passwordOf('l1@spot.net')
assert.deepEqual([password, source, spot.status, spot.description], ['fromnzb1', 'nzb', 'listed', 'Plot'])
assert.isNotNull(spot.nzb_read_at)
assert.equal((await db.from('releases').where('id', spot.release_id).first()).file_count, 0)
// Read once: nothing left.
assert.deepEqual(await SpotService.readListings(10, 2), {})
})
test("the spot's text when the NZB has none; the NZB's own password wins over it", async ({ assert }) => {
await listedSpot('l2@spot.net', { description: 'Wachtwoord: fromtext2', nzb: nzb(null) })
assert.deepEqual(await SpotService.readListings(10, 1), { 'spot-password': 1 })
assert.include(await passwordOf('l2@spot.net'), { password: 'fromtext2', source: 'spot' })
await listedSpot('l3@spot.net', { description: 'Wachtwoord: fromtext3', nzb: nzb('fromnzb3') })
await db.from('releases').where('id', (await db.from('spots').where('message_id', 'l3@spot.net').first()).release_id).update({ password: 'fromtext3', password_source: 'spot' })
assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-password': 1 })
assert.include(await passwordOf('l3@spot.net'), { password: 'fromnzb3', source: 'nzb' })
})
test('a placeholder NZB hides the listing; a lost connection gives the spot back, a gone article does not', async ({ assert }) => {
await listedSpot('l4@spot.net', { nzb: nzb(null, 1500) })
assert.deepEqual(await SpotService.readListings(10, 1), { dummy: 1 })
assert.equal((await db.from('spots').where('message_id', 'l4@spot.net').first()).status, 'dummy')
await listedSpot('l5@spot.net', { error: 'Connection reset' })
assert.deepEqual(await SpotService.readListings(10, 1), { retry: 1 })
assert.isNull((await db.from('spots').where('message_id', 'l5@spot.net').first()).nzb_read_at)
await db.from('spots').where('message_id', 'l5@spot.net').delete()
await listedSpot('l6@spot.net', { error: '430 No Such Article' })
assert.deepEqual(await SpotService.readListings(10, 1), { gone: 1 })
assert.isNotNull((await db.from('spots').where('message_id', 'l6@spot.net').first()).nzb_read_at)
// A corrupt NZB is a result, not something to read again.
await listedSpot('l7@spot.net', { nzbError: new NzbUnreadable('invalid code lengths set') })
assert.deepEqual(await SpotService.readListings(10, 1), { 'nzb-unreadable': 1 })
const l7 = await db.from('spots').where('message_id', 'l7@spot.net').first()
assert.isNotNull(l7.nzb_read_at)
assert.equal(l7.error, 'NZB unreadable: invalid code lengths set')
})
})
tests/functional/staging.spec.ts
-70
View File
@@ -1,70 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import StagingService from '#services/StagingService'
import IngestService, { segmentFromHeader } from '#services/IngestService'
const segment = (name: string, k: number, n: number, id = `${name}-${k}@x`) =>
segmentFromHeader(
{ 'subject': `"${name}" yEnc (${k}/${n})`, 'from': 'poster@x', 'message-id': `<${id}>`, 'date': 'Thu, 01 Oct 2026 12:00:00 GMT', ':bytes': '700000' },
'alt.binaries.test',
k
)!
async function binary(name: string) {
return db.from('binaries').where('name', name).first()
}
test.group('Staging', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('a lone segment waits; its second segment brings both in', async ({ assert }) => {
let r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
assert.equal(r.staged, 1)
assert.lengthOf(r.ingest, 0)
await IngestService.ingest(r.ingest)
assert.isNull(await binary('rnd1.bin'))
// The same article again (a re-fetched range) is no second segment.
r = await StagingService.filter([segment('rnd1.bin', 3, 50)])
assert.lengthOf(r.ingest, 0)
// Nor is the same article crossposted to another group (other group, other article number).
const crosspost = { ...segment('rnd1.bin', 3, 50), groups: ['alt.binaries.boneless'], articleNumber: 160154327571 }
r = await StagingService.filter([crosspost])
assert.lengthOf(r.ingest, 0)
r = await StagingService.filter([segment('dup.bin', 2, 9), { ...segment('dup.bin', 2, 9), groups: ['alt.binaries.boneless'] }])
assert.equal(r.staged, 1)
r = await StagingService.filter([segment('rnd1.bin', 4, 50)])
assert.deepEqual(r.ingest.map((s) => s.number).sort(), [3, 4])
await IngestService.ingest(r.ingest)
const b = await binary('rnd1.bin')
assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 2)
assert.lengthOf(await db.from('staged_segments').where('file_key', segment('rnd1.bin', 3, 50).fileKey), 0)
// Later segments of a known file go straight in.
r = await StagingService.filter([segment('rnd1.bin', 9, 50)])
assert.lengthOf(r.ingest, 1)
})
test('single-part files and files with two segments in the batch go straight in; staged ones join them', async ({ assert }) => {
// Part 1 arrived alone at the end of the previous batch.
await StagingService.filter([segment('real.part01.rar', 1, 3)])
const r = await StagingService.filter([segment('real.part01.rar', 2, 3), segment('real.part01.rar', 3, 3), segment('one.nfo', 1, 1)])
assert.equal(r.staged, 0)
assert.sameMembers(r.ingest.map((s) => `${s.name}:${s.number}`), ['real.part01.rar:1', 'real.part01.rar:2', 'real.part01.rar:3', 'one.nfo:1'])
})
test('GC adopts segments whose file got a binary, and drops old ones', async ({ assert }) => {
await StagingService.filter([segment('late.bin', 7, 10)])
await StagingService.filter([segment('old.bin', 2, 10)])
await db.from('staged_segments').where('file_key', segment('old.bin', 2, 10).fileKey).update({ staged_at: new Date(Date.now() - 3 * 3600_000) })
// Another job created the binary meanwhile.
await IngestService.ingest([segment('late.bin', 1, 10), segment('late.bin', 2, 10)])
const result = await StagingService.sweep(2 * 3600, Date.now() + 10_000)
assert.deepEqual(result, { adopted: 1, expired: 1 })
const b = await binary('late.bin')
assert.equal(Number((await db.from('parts').where('binary_id', b.id).count('* as n'))[0].n), 3)
})
})
tests/functional/web_api.spec.ts
-145
View File
@@ -1,145 +0,0 @@
import { test } from '@japa/runner'
import ReleaseCompleteness from '#services/ReleaseCompleteness'
import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import testUtils from '@adonisjs/core/services/test_utils'
async function seed() {
const now = new Date()
const [release] = await db
.table('releases')
.insert({ release_key: 'w'.repeat(40), name: 'Webtest.Show.S01E02.1080p', source: 'par2', groups: '["alt.binaries.test"]', posted_at: now, first_seen_at: now, last_seen_at: now, contents: JSON.stringify([{ name: 'Webtest.Show.S01E02.1080p/episode.mkv', size: 1000 }]) })
.returning('id')
await FileStore.insert([{
filename: 'Xq8r2KdLm3Zw91PqA7webtest', real_name: 'Webtest.Show.S01E02.1080p.mkv', poster: 'p@example.invalid',
date: now.getTime(), parts: 2, size: 1400, release_id: release.id, groups: '["alt.binaries.test"]',
segments: [{ number: 1, id: 'one@test', size: 700 }, { number: 2, id: 'two@test', size: 700 }],
created_at: now, updated_at: now,
}])
return Number(release.id)
}
test.group('Web API', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('searches by words in any order, with dots and spaces interchangeable', async ({ client, assert }) => {
const id = await seed()
for (const q of ['webtest show s01e02', 'Webtest.Show', 'episode.mkv 1080p']) {
const response = await client.get('/api/v1/releases').qs({ q })
response.assertStatus(200)
assert.include(response.body().releases.map((r: any) => r.id), id, q)
}
const none = await client.get('/api/v1/releases').qs({ q: 'webtest nosuchword' })
assert.notInclude(none.body().releases.map((r: any) => r.id), id)
})
test('filters and details', async ({ client, assert }) => {
const id = await seed()
const readable = await client.get('/api/v1/releases').qs({ q: 'webtest', readable: '1', source: 'par2' })
assert.include(readable.body().releases.map((r: any) => r.id), id)
const other = await client.get('/api/v1/releases').qs({ q: 'webtest', source: 'rar' })
assert.notInclude(other.body().releases.map((r: any) => r.id), id)
const detail = await client.get(`/api/v1/releases/${id}`)
detail.assertStatus(200)
assert.equal(detail.body().files[0].name, 'Webtest.Show.S01E02.1080p.mkv')
assert.equal(detail.body().fileCount, 1)
assert.equal(detail.body().size, 1400)
;(await client.get('/api/v1/releases/999999999')).assertStatus(404)
})
test('filters by size in MB and by file extension as a whole word', async ({ client, assert }) => {
const id = await seed() // 1400 bytes, an episode.mkv in its listing
const ids = async (qs: Record<string, string>) =>
(await client.get('/api/v1/releases').qs({ q: 'webtest', ...qs })).body().releases.map((r: any) => r.id)
assert.include(await ids({ min_size: '0.001', max_size: '0.002' }), id)
assert.notInclude(await ids({ min_size: '1' }), id)
assert.notInclude(await ids({ max_size: '0.001' }), id)
assert.include(await ids({ ext: 'mkv' }), id)
assert.include(await ids({ ext: '.iso, MKV' }), id)
assert.notInclude(await ids({ ext: 'iso' }), id)
// "sho" is part of "show", not an extension.
assert.notInclude(await ids({ ext: 'sho' }), id)
// A plainly posted file counts too, not only archive entries; volumes count as their archive.
const now = new Date()
await FileStore.insert([
{ filename: 'Webtest.Show.S01E02.1080p.nfo', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now },
{ filename: 'Webtest.Show.S01E02.1080p.7z.002', poster: 'p', date: now.getTime(), parts: 1, size: 10, release_id: id, groups: '[]', segments: [], created_at: now, updated_at: now },
])
assert.include(await ids({ ext: 'nfo' }), id)
assert.include(await ids({ ext: '7z' }), id)
assert.deepEqual((await db.from('releases').where('id', id).first()).extensions, ['7z', 'mkv', 'nfo'])
})
test('serves an NZB with the real file name', async ({ client, assert }) => {
const id = await seed()
const response = await client.get(`/api/v1/releases/${id}/nzb`)
response.assertStatus(200)
assert.include(response.header('content-type'), 'application/x-nzb')
assert.include(response.header('content-disposition'), 'Webtest.Show.S01E02.1080p.nzb')
assert.include(response.text(), '&quot;Webtest.Show.S01E02.1080p.mkv&quot; yEnc (1/2)')
assert.include(response.text(), '<segment bytes="700" number="1">one@test</segment>')
})
test('stats', async ({ client }) => {
const response = await client.get('/api/v1/stats')
response.assertStatus(200)
response.assertBodyContains({ sources: [] })
})
test('completeness follows files and par2 lists (triggers note releases, the sweep recomputes)', async ({ assert }) => {
const now = new Date()
const complete = async (id: number) => {
await ReleaseCompleteness.sweep()
return (await db.from('releases').where('id', id).first()).complete
}
const file = (releaseId: number, name: string) => ({
filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId,
groups: '[]', segments: [], created_at: now, updated_at: now,
})
// Counter rule: [i/M] announced 2 files.
const [counted] = await db.table('releases').insert({ release_key: 'c'.repeat(40), name: 'Counted', groups: '[]', file_total: 2, first_seen_at: now, last_seen_at: now }).returning('id')
await FileStore.insert([file(counted.id, 'a.rar')])
assert.isFalse(await complete(counted.id))
await FileStore.insert([file(counted.id, 'b.rar')])
assert.isTrue(await complete(counted.id))
// par2 rule: the listed file becomes present once a file of its exact size is there.
const [listed] = await db.table('releases').insert({ release_key: 'l'.repeat(40), name: 'Listed', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id')
await db.table('release_files').insert({ release_id: listed.id, par2_file_id: '1'.repeat(32), name: 'Listed.mkv', size: 5_000_000, md5_16k: '0'.repeat(32) })
assert.isFalse(await complete(listed.id))
const [f] = await FileStore.insert([file(listed.id, 'Xq8r2KdLm3Zw91PqA7')])
const [b] = await db.table('binaries').insert({ file_key: 'z'.repeat(40), poster: 'p', subject: '"x" yEnc', name: 'x', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, file_id: f.id }).returning('id')
assert.isFalse(await complete(listed.id))
await db.from('binaries').where('id', b.id).update({ size: 5_000_000 })
assert.isTrue(await complete(listed.id))
})
test('par2-only releases are hidden unless asked for', async ({ client, assert }) => {
const now = new Date()
const [release] = await db.table('releases')
.insert({ release_key: 'p'.repeat(40), name: 'Webtest.Par2only.S01E01.1080p.WEB-DL-GRP', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
await db.table('release_files').insert({ release_id: release.id, par2_file_id: '2'.repeat(32), name: 'Webtest.Par2only.S01E01.mkv', size: 1_000_000_000, md5_16k: '0'.repeat(32) })
await FileStore.insert([{ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919,
release_id: release.id, groups: '[]', segments: [], created_at: now, updated_at: now }])
const hidden = await client.get('/api/v1/releases').qs({ q: 'webtest par2only' })
assert.notInclude(hidden.body().releases.map((r: any) => r.id), Number(release.id))
const shown = await client.get('/api/v1/releases').qs({ q: 'webtest par2only', par2_only: '1' })
const row = shown.body().releases.find((r: any) => r.id === Number(release.id))
assert.isTrue(row.par2Only)
})
})
test.group('Health check', () => {
test('/health answers when the database does', async ({ client }) => {
const response = await client.get('/health')
response.assertStatus(200)
response.assertBody({ ok: true })
})
})
tests/functional/yenc_name.spec.ts
-40
View File
@@ -1,40 +0,0 @@
import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import FileStore from '#services/FileStore'
import ContentService from '#services/ContentService'
let keys = 0
async function counterFile(name: string) {
const now = new Date()
const [r] = await db
.table('releases')
.insert({ release_key: `yenc${++keys}`.padEnd(40, 'y'), name, source: 'weak-stem', groups: '[]', first_seen_at: now, last_seen_at: now })
.returning('id')
const [{ id }] = await FileStore.insert([
{ filename: name, poster: 'bob@home.com', date: now.getTime(), parts: 2, size: 10, release_id: r.id, groups: '["alt.binaries.boneless"]',
segments: [{ number: 1, id: `${name}@nyuu`, size: 10 }], created_at: now, updated_at: now },
])
return db.from('files').where('id', id).first()
}
test.group('Names from the yEnc header', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('random-named files take the yEnc name and gather in its release', async ({ assert }) => {
const a = await counterFile('P9BSPl59iDfQUVvXFeBVG4MDQaOTufds')
const b = await counterFile('Faoq6nlEaVlTzYyNA27sSfFm2KvLHOvD')
const nameFromYenc = (ContentService as any).nameFromYenc.bind(ContentService)
assert.equal(await nameFromYenc(a, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar'), 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part01.rar')
await nameFromYenc(b, null, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP.part02.rar')
const [fa, fb] = await Promise.all([db.from('files').where('id', a.id).first(), db.from('files').where('id', b.id).first()])
assert.equal(fa.release_id, fb.release_id)
assert.equal((await db.from('releases').where('id', fa.release_id).first()).name, 'Deep.Strike.S01E16.1080p.WEB.h264-GRP')
// A random yEnc name, or a file that has a real name already, stays as it is.
const c = await counterFile('Vg9QflfNLXVMPOWtK4KRJv474DadgTIF')
assert.isUndefined(await nameFromYenc(c, null, 'qTcrIl1qTo4Bx6El5c4RtTA4ZSqwd6tz'))
const d = await counterFile('Some.Release.part01.rar')
assert.isUndefined(await nameFromYenc(d, null, 'Other.Name.part01.rar'))
})
})
tests/unit/aggregate.spec.ts
-56
View File
@@ -1,56 +0,0 @@
import { test } from '@japa/runner'
import { FileAggregator, type AggregatableSegment } from '#lib/aggregate/index'
const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
fileKey,
number,
totalParts,
messageId,
bytes: 100,
})
test.group('aggregate / FileAggregator', () => {
test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
const aggregator = new FileAggregator()
assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
assert.equal(aggregator.openFiles, 1)
const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
assert.equal(done.fileKey, 'a')
assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
assert.equal(aggregator.openFiles, 0)
})
test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
const aggregator = new FileAggregator()
const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
})
test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
const aggregator = new FileAggregator()
aggregator.add([seg('old', 1, 3)])
aggregator.add([seg('new', 1, 3)])
aggregator.add([seg('new', 2, 3)])
assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
assert.equal(aggregator.openFiles, 0)
})
})
test.group('seed / runRange', () => {
test('a range already past its end only reports that it is done', async ({ assert }) => {
const { runRange } = await import('#lib/seed/runner')
const batches: any[] = []
await runRange({
client: { group: async () => assert.fail('no GROUP'), xover: async () => assert.fail('no XOVER') },
group: 'a.b.test',
start: 101,
end: 100,
sink: async (batch) => {
batches.push(batch)
},
})
assert.deepEqual(batches.map((b) => [b.next, b.headers, b.files.length]), [[101, 0, 0]])
})
})
tests/unit/content.spec.ts
-65
View File
@@ -1,65 +0,0 @@
import { test } from '@japa/runner'
import yencode from 'yencode'
import { isLaterVolume, sniffContentType } from '#lib/content/index'
import { crc32Hex, decodeSegment, YencError } from '#lib/yenc/index'
test.group('content / sniffContentType', () => {
test('recognises containers and media by magic bytes', ({ assert }) => {
const magic = (hex: string, pad = 16) => Buffer.concat([Buffer.from(hex, 'hex'), Buffer.alloc(pad)])
assert.equal(sniffContentType(Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(40)])), 'nzbe')
assert.equal(sniffContentType(magic('5041523200504b54')), 'par2')
assert.equal(sniffContentType(magic('526172211a070100')), 'rar5')
assert.equal(sniffContentType(magic('526172211a0700')), 'rar4')
assert.equal(sniffContentType(magic('1a45dfa3')), 'matroska')
assert.equal(sniffContentType(magic('0000001866747970')), 'mp4')
assert.equal(sniffContentType(magic('28b52ffd')), 'zstd')
assert.equal(sniffContentType(Buffer.from('<?xml version="1.0"?>\n<!DOCTYPE nzb PUBLIC "x">')), 'nzb')
assert.isNull(sniffContentType(Buffer.from('plain text')))
assert.equal(sniffContentType(magic('ffd8ffe0')), 'jpeg')
assert.equal(sniffContentType(magic('89504e470d0a1a0a')), 'png')
assert.equal(sniffContentType(Buffer.concat([Buffer.from('ID3'), Buffer.alloc(16)])), 'mp3')
assert.equal(sniffContentType(Buffer.concat([Buffer.from('fLaC'), Buffer.alloc(16)])), 'flac')
assert.equal(sniffContentType(Buffer.from('RIFF\x00\x00\x00\x00AVI LIST', 'latin1')), 'avi')
// UDF (Blu-ray) volume recognition sequence at 32 KiB
const udf = Buffer.alloc(0x9000)
udf.write('BEA01', 0x8001, 'latin1')
udf.write('NSR03', 0x8801, 'latin1')
assert.equal(sniffContentType(udf), 'udf')
})
})
test.group('yenc / lenientSize', () => {
test("recognises yEncBin's encrypted .bin container by its header", ({ assert }) => {
// The first 48 bytes of 2141406641.bin (yEncBin@Poster.com).
const data = Buffer.from('54f399d117c492696e4dc4ef5b60c11d7f83e92dfdf23450e79370b048123da81e4c8013091b682fb22478daed2ae981', 'hex')
assert.equal(sniffContentType(data), 'yencbin')
// The second variant (1060238685.bin).
assert.equal(sniffContentType(Buffer.from('54f399e917c492696a4e9fb05b60c31d7f83e92df9f16f5086e3c3d9480d66ef1e48d915', 'hex')), 'yencbin')
assert.isNull(sniffContentType(Buffer.from('54f399d117c492696e4dc4ef5b60c11e7f83e92d', 'hex')))
})
test('decodes a part whose declared size leaves out a header, still checking the CRC', ({ assert }) => {
const data = Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(16, 7), Buffer.from('ciphertext-here')])
const declared = data.length - 20
const body = Buffer.from(
`=ybegin part=1 line=128 size=${declared} name=f6dd6b156692a52b.bin\r\n=ypart begin=1 end=${declared}\r\n` +
yencode.encode(data).toString('latin1') +
`\r\n=yend size=${declared} part=1 pcrc32=${crc32Hex(data)}`,
'latin1'
)
assert.throws(() => decodeSegment(body), YencError as any)
assert.isTrue(decodeSegment(body, { lenientSize: true }).data.equals(data))
})
})
test.group('content / isLaterVolume', () => {
test('recognises later RAR volumes and par2 recovery volumes, also under a random stem', ({ assert }) => {
for (const name of ['P8vttYaPec1mL4nDZHPZo7Nd6iEWj8lP.part47.rar', 'Show.S01E01.part002.rar', 'abc.r00', 'abc.r123', 'x7Kq9.vol03+04.par2', 'Show.vol127-128.par2']) {
assert.isTrue(isLaterVolume(name), name)
}
for (const name of ['P8vttYaPec1mL4nDZHPZo7Nd6iEWj8lP.part01.rar', 'abc.part1.rar', 'abc.rar', 'x7Kq9.par2', '6e13186238ab54ca498e46d2d77da099', 'abc.7z.002', 'abc.mkv']) {
assert.isFalse(isLaterVolume(name), name)
}
})
})
tests/unit/date_search.spec.ts
-14
View File
@@ -1,14 +0,0 @@
import { test } from '@japa/runner'
import { articleAtDate } from '#lib/nntp/date_search'
test.group('nntp / articleAtDate', () => {
test('finds the article posted at a time, skipping gaps', async ({ assert }) => {
// One article a second from t0, articles 1,000,000–1,100,000; 1,040,000–1,040,999 expired.
const t0 = Date.UTC(2026, 8, 1)
const dateOf = async (n: bigint) => (n >= 1_040_000n && n < 1_041_000n ? null : t0 + Number(n - 1_000_000n) * 1000)
const at = await articleAtDate(1_000_000n, 1_100_000n, t0 + 60_000 * 1000, dateOf, 100n)
assert.isAtLeast(Number(at), 1_059_900)
assert.isAtMost(Number(at), 1_060_000)
assert.equal(await articleAtDate(1_000_000n, 1_100_000n, t0 - 1, dateOf, 100n), 1_000_000n)
})
})
tests/unit/delimited_reader.spec.ts
-52
View File
@@ -1,52 +0,0 @@
import { test } from '@japa/runner'
import { delimitedReader } from '#lib/nntp/delimited_reader'
const CRLF = Buffer.from('\r\n')
const END = Buffer.from('\r\n.\r\n')
async function* chunks(...parts: string[]) {
for (const part of parts) yield Buffer.from(part, 'latin1')
}
test.group('nntp / delimitedReader', () => {
test('reads an empty multi-line reply (XOVER of a range without articles)', async ({ assert }) => {
const reader = delimitedReader(chunks('224 Overview follows\r\n.\r\n', '211 1 2 3 g\r\n'), CRLF)
assert.equal((await reader.next()).value!.toString(), '224 Overview follows')
assert.equal((await reader.next(END)).value!.toString(), '')
assert.equal((await reader.next(CRLF)).value!.toString(), '211 1 2 3 g')
})
test('reads multi-line data, also with the terminator split over chunks', async ({ assert }) => {
for (const split of [1, 2, 3, 4, 5, 6, 7]) {
const reply = '224 ok\r\nline 1\r\n..dotted\r\nline 3\r\n.\r\n200 next\r\n'
const at = reply.indexOf('.\r\n200') - 2 + split - 1
const reader = delimitedReader(chunks(reply.slice(0, at), reply.slice(at)), CRLF)
assert.equal((await reader.next()).value!.toString(), '224 ok')
assert.equal((await reader.next(END)).value!.toString(), 'line 1\r\n..dotted\r\nline 3', `split ${split}`)
assert.equal((await reader.next(CRLF)).value!.toString(), '200 next')
}
})
test('reads replies byte by byte', async ({ assert }) => {
const text = '200 welcome\r\n224 ok\r\na\r\n.\r\n224 ok\r\n.\r\n211 x\r\n'
const reader = delimitedReader(chunks(...text.split('')), CRLF)
assert.equal((await reader.next()).value!.toString(), '200 welcome')
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
assert.equal((await reader.next(END)).value!.toString(), 'a')
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
assert.equal((await reader.next(END)).value!.toString(), '')
assert.equal((await reader.next(CRLF)).value!.toString(), '211 x')
})
test('keeps binary data intact', async ({ assert }) => {
const body = Buffer.from([0, 13, 255, 10, 46, 13, 46, 10, 13])
const source = (async function* () {
yield Buffer.from('222 body\r\n')
yield body
yield Buffer.from('\r\n.\r\n')
})()
const reader = delimitedReader(source, CRLF)
await reader.next()
assert.deepEqual((await reader.next(END)).value, body)
})
})
tests/unit/nicovideo.spec.ts
-46
View File
@@ -1,46 +0,0 @@
import { test } from '@japa/runner'
import { randomBytes } from 'node:crypto'
import { zstdCompressSync, constants } from 'node:zlib'
import { NICOVIDEO_FILE, nicovideoInfo, nicovideoReleaseName, zstdPrefix } from '#lib/content/index'
function tarEntry(name: string, data: Buffer): Buffer {
const header = Buffer.alloc(512)
header.write(name, 0, 'utf8')
header.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 'latin1')
header.write('ustar\0', 257, 'latin1')
return Buffer.concat([header, data, Buffer.alloc((512 - (data.length % 512)) % 512)])
}
test.group('nicovideo captures', () => {
test('reads the title from the start of a capture, even a truncated one', async ({ assert }) => {
const dir = '1790720489_nicovideo_jp_watch_sm27965722'
const response = {
data: {
metadata: { jsonLds: [{ author: { name: 'まふまふ' } }] },
response: {
video: { id: 'sm27965722', title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】', registeredAt: '2016-01-08T20:00:00+09:00' },
owner: { nickname: 'まふまふ' },
tag: { items: [{ name: '音楽' }, { name: 'After_the_Rain' }] },
},
},
}
const tar = Buffer.concat([
tarEntry(`${dir}/`, Buffer.alloc(0)),
tarEntry(`${dir}/server_response.metadata`, Buffer.from(JSON.stringify(response, null, 2))),
tarEntry(`${dir}/01.cmfa`, randomBytes(300_000)),
])
const compressed = zstdCompressSync(tar, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
// Only the first part of the stream, as in the file's first article.
const info = nicovideoInfo(await zstdPrefix(compressed.subarray(0, Math.floor(compressed.length * 0.6))))!
assert.deepEqual(info, {
id: 'sm27965722',
title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】',
owner: 'まふまふ',
registeredAt: '2016-01-08T20:00:00+09:00',
tags: ['音楽', 'After_the_Rain'],
})
assert.equal(nicovideoReleaseName(info, 'sm1'), '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】 - まふまふ (nicovideo sm27965722)')
assert.equal(NICOVIDEO_FILE.exec(`${dir}.tar.zst`)![1], 'sm27965722')
assert.isNull(nicovideoInfo(await zstdPrefix(Buffer.from('not zstd'))))
})
})
tests/unit/nntp_group.spec.ts
-18
View File
@@ -1,18 +0,0 @@
import { test } from '@japa/runner'
import { parseGroupResponse } from '#lib/nntp/group'
test.group('nntp / parseGroupResponse', () => {
test('parses the reply from news.eweka.nl', ({ assert }) => {
// Captured 2026-09-27. count == last - first + 1, so the field order is standard.
assert.deepEqual(parseGroupResponse('211 27275164317 266599 27275430915 alt.binaries.test'), {
count: 27275164317n,
first: 266599n,
last: 27275430915n,
name: 'alt.binaries.test',
})
})
test('rejects non-211 replies', ({ assert }) => {
assert.throws(() => parseGroupResponse('411 No such group'), /Unexpected GROUP response/)
})
})
tests/unit/nzb.spec.ts
-67
View File
@@ -1,67 +0,0 @@
import { test } from '@japa/runner'
import { buildNzb } from '#lib/nzb/index'
test.group('nzb / buildNzb', () => {
test('writes files, groups and ordered segments', ({ assert }) => {
const xml = buildNzb(
[
{
name: 'Some.Show.S01E01.mkv',
poster: 'poster@example.invalid',
date: 1_790_000_000_000,
groups: ['alt.binaries.test'],
segments: [{ number: 2, id: 'b@x', size: 700 }, { number: 1, id: 'a@x', size: 800 }],
},
],
{ title: 'Some.Show.S01E01' }
)
assert.include(xml, '<!DOCTYPE nzb PUBLIC "-//newzBin//DTD NZB 1.1//EN"')
assert.include(xml, '<meta type="name">Some.Show.S01E01</meta>')
assert.include(xml, 'subject="&quot;Some.Show.S01E01.mkv&quot; yEnc (1/2)"')
assert.include(xml, 'date="1790000000"')
assert.include(xml, '<group>alt.binaries.test</group>')
assert.isBelow(xml.indexOf('>a@x<'), xml.indexOf('>b@x<'))
assert.include(xml, '<segment bytes="800" number="1">a@x</segment>')
})
})
test.group('nzb / parseNzb', () => {
test('reads meta, files, groups and segments, decoding entities', async ({ assert }) => {
const { parseNzb } = await import('#lib/nzb/index')
const xml = `<?xml version="1.0" encoding="utf-8" ?>
<!DOCTYPE nzb PUBLIC "-//newzBin//DTD NZB 0.9//EN" "http://www.newzbin.com/DTD/nzb/nzb-0.9.dtd">
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
<head><meta type="name">Some.Show.S01E01</meta><meta type="password">s3cr&amp;t</meta></head>
<file poster="Yenc@power-post.org (Cosmo71)" date="1790598923" subject="Wond40 &#x22;Nachbe&#x22; x.part1.rar&#x22; yEnc (0001/2)">
<groups><group>alt.binaries.bloaf</group></groups>
<segments>
<segment bytes="1000" number="2">b@x.local</segment>
<segment bytes="2000" number="1">&lt;a@x.local&gt;</segment>
</segments>
</file>
<file poster="p" date="1" subject="empty"><groups/><segments></segments></file>
</nzb>`
const nzb = parseNzb(xml)!
assert.deepEqual(nzb.meta, { name: 'Some.Show.S01E01', password: 's3cr&t' })
assert.lengthOf(nzb.files, 1)
assert.equal(nzb.files[0].subject, 'Wond40 "Nachbe" x.part1.rar" yEnc (0001/2)')
assert.deepEqual(nzb.files[0].groups, ['alt.binaries.bloaf'])
assert.deepEqual(nzb.files[0].segments, [
{ number: 1, bytes: 2000, id: 'a@x.local' },
{ number: 2, bytes: 1000, id: 'b@x.local' },
])
assert.isNull(parseNzb('<html>not an nzb</html>'))
})
})
test.group('nzb / stored segment lists', () => {
test('pack and unpack round-trip, in order, also with gaps in the numbering', async ({ assert }) => {
const { packSegments, unpackSegments } = await import('#lib/nzb/segments')
const sequential = Array.from({ length: 300 }, (_, i) => ({ number: i + 1, id: `Ab${i}Cd-1790600369${i}@nyuu`, size: i === 299 ? 1234 : 740000 }))
assert.deepEqual(unpackSegments(packSegments([...sequential].reverse())), sequential)
const gaps = [{ number: 3, id: 'c@x', size: 3 }, { number: 1, id: 'a@x', size: 1 }, { number: 7, id: 'g@x', size: 7 }]
assert.deepEqual(unpackSegments(packSegments(gaps)), [gaps[1], gaps[0], gaps[2]])
assert.deepEqual(unpackSegments(packSegments([])), [])
assert.isBelow(packSegments(sequential).length / sequential.length, 30)
})
})
tests/unit/par2.spec.ts
-60
View File
@@ -1,60 +0,0 @@
import { test } from '@japa/runner'
import { createHash } from 'node:crypto'
import { isPar2, parsePar2 } from '#lib/par2/index'
const md5 = (data: Buffer) => createHash('md5').update(data).digest()
/** Builds one PAR2 packet with a valid checksum. */
function packet(setId: Buffer, type: string, body: Buffer) {
const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
const length = Buffer.alloc(8)
length.writeBigUInt64LE(BigInt(64 + padded.length))
const typeBuf = Buffer.alloc(16)
typeBuf.write(type, 'latin1')
const hashed = Buffer.concat([setId, typeBuf, padded])
return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
}
function fileDesc(setId: Buffer, name: string, size: number) {
const len = Buffer.alloc(8)
len.writeBigUInt64LE(BigInt(size))
const md5_16k = md5(Buffer.from(`${name}-16k`))
const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
return packet(setId, 'PAR 2.0\0FileDesc', body)
}
test.group('par2 / parsePar2', () => {
const setId = Buffer.alloc(16, 7)
test('reads file names and sizes from FileDesc packets', ({ assert }) => {
const data = Buffer.concat([
packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)),
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600),
fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800),
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common
])
assert.isTrue(isPar2(data))
const set = parsePar2(data)!
assert.equal(set.recoverySetId, setId.toString('hex'))
assert.deepEqual(
set.files.map((f) => [f.name, f.length]),
[
['Some.Movie.2024.part01.rar', 104857600],
['Some.Movie.2024.part02.rar', 52428800],
]
)
})
test('skips corrupted packets and resumes at the next one', ({ assert }) => {
const bad = fileDesc(setId, 'bad.bin', 1)
bad[100] ^= 0xff
const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)])
assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin'])
})
test('returns null for non-par2 data', ({ assert }) => {
assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1')))
assert.isNull(parsePar2(Buffer.from('hello world')))
})
})
tests/unit/probe.spec.ts
-86
View File
@@ -1,86 +0,0 @@
import { test } from '@japa/runner'
import { crc32Hex } from '#lib/yenc/index'
import { probeInspection, type InspectTask } from '#lib/content/probe'
/** One yEnc article of `file`: part `index` of `size` bytes (escaping '.' and whitespace too, so no dot-stuffing). */
function yencArticle(name: string, file: Buffer, index: number, size: number) {
const data = file.subarray(index * size, (index + 1) * size)
const total = Math.ceil(file.length / size)
const out: number[] = []
for (const byte of data) {
const c = (byte + 42) & 0xff
if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
else out.push(c)
}
const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
return Buffer.from(head + Buffer.from(out).toString('latin1') + tail, 'latin1')
}
/** A posted file as articles, and its inspection task. */
function posted(name: string, file: Buffer, size = 400) {
const articles = new Map<string, Buffer>()
const segments = []
for (let i = 0; i * size < file.length; i++) {
const id = `${name}-${i}@test`
articles.set(id, yencArticle(name, file, i, size))
segments.push({ number: i + 1, id, size: size + 100 })
}
const fetched: string[] = []
const fetch = async (id: string) => {
fetched.push(id)
const body = articles.get(id)
if (!body) throw new Error(`430 no such article ${id}`)
return body
}
const task: InspectTask = { kind: 'inspect', fileId: 1, name, segments, sevenZip: { volumes: 1, last: segments } }
return { task, fetch, fetched }
}
// `7z a -pSecret -mhe=on`: an encrypted header (tests/unit/sevenzip.spec.ts).
const HEADER_ENCRYPTED = Buffer.from(
'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
'base64'
)
test.group('content probe', () => {
test('a first segment gives size, fingerprint and the yEnc name', async ({ assert }) => {
const file = Buffer.alloc(20_000, 7)
const { task, fetch, fetched } = posted('Some.Name.bin', file, 17_000)
const probe = await probeInspection(task, fetch)
assert.include(probe, { size: 20_000, contentType: null, yencName: 'Some.Name.bin' })
assert.match(probe.md5_16k!, /^[0-9a-f]{32}$/)
assert.deepEqual(fetched, ['Some.Name.bin-0@test'])
})
test('an NZB is read whole', async ({ assert }) => {
const xml = `<?xml version="1.0"?>\n<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${'<file subject="x"/>'.repeat(60)}</nzb>`
const { task, fetch, fetched } = posted('x.nzb', Buffer.from(xml))
const probe = await probeInspection(task, fetch)
assert.equal(probe.contentType, 'nzb')
assert.equal(probe.text, xml)
// Every segment once: the first, decoded to sniff, is reused for the whole.
assert.deepEqual(fetched, task.segments.map((s) => s.id))
})
test('a 7z archive with encrypted headers is known from its tail, without bsdtar', async ({ assert }) => {
const { task, fetch } = posted('a.7z', HEADER_ENCRYPTED, 100)
const probe = await probeInspection(task, fetch)
assert.equal(probe.contentType, '7z')
assert.include(probe.archive!, { format: '7z', headersEncrypted: true })
assert.lengthOf(probe.archive!.entries, 0)
})
test('a missing first article is a result, not a failure (no retries)', async ({ assert }) => {
const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1))
task.segments[0].id = 'missing@test'
assert.include(await probeInspection(task, fetch), { contentType: 'missing' })
// Through the seed relay: 410.
const relayed = await probeInspection(task, async () => {
throw new Error('Seed API /api/v1/seed/articles?id=x: 410 {"error":"No such article"}')
})
assert.equal(relayed.contentType, 'missing')
// Anything else still fails (and is retried).
await assert.rejects(() => probeInspection(task, async () => { throw new Error('NNTP BODY: no answer within 120 s') }), /no answer/)
})
})
tests/unit/rar.spec.ts
-376
View File
@@ -1,376 +0,0 @@
import { test } from '@japa/runner'
import { crc32 } from 'node:zlib'
import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, walkRar, type RarEntry, type VolumeReader } from '#lib/rar/index'
import { crc32Hex } from '#lib/yenc/index'
/** Entries without the extraction fields, for comparing listings. */
const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
// --- RAR 4 builders -----------------------------------------------------------
const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
function rar4Block(type: number, flags: number, body: Buffer) {
const header = Buffer.alloc(7)
header.writeUInt8(type, 2)
header.writeUInt16LE(flags, 3)
header.writeUInt16LE(7 + body.length, 5)
const block = Buffer.concat([header, body])
block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0)
return block
}
function rar4Main(flags: number) {
return rar4Block(0x73, flags, Buffer.alloc(6))
}
function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) {
const body = Buffer.alloc(25)
body.writeUInt8(opts.method ?? 0x33, 18)
body.writeUInt32LE(opts.packed, 0)
body.writeUInt32LE(opts.unpacked, 4)
body.writeUInt16LE(name.length, 19)
// LONG_BLOCK (0x8000) is always set on file headers.
return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)])
}
// --- RAR 5 builders -----------------------------------------------------------
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
function vint(value: number) {
const bytes: number[] = []
do {
let byte = value % 128
value = Math.floor(value / 128)
if (value > 0) byte |= 0x80
bytes.push(byte)
} while (value > 0)
return Buffer.from(bytes)
}
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
let flags = headerFlags
const parts = [vint(type)]
const sizes: Buffer[] = []
if (opts.extra) {
flags |= 0x0001
sizes.push(vint(opts.extra.length))
}
if (opts.data !== undefined) {
flags |= 0x0002
sizes.push(vint(opts.data))
}
const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
const sized = Buffer.concat([vint(header.length), header])
const crc = Buffer.alloc(4)
crc.writeUInt32LE(crc32(sized) >>> 0)
return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
}
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
const nameBuf = Buffer.from(name, 'utf8')
const fields = Buffer.concat([
vint(opts.dir ? 0x0001 : 0), // file flags
vint(size),
vint(0x20), // attributes
vint(0), // compression info
vint(0), // host OS
vint(nameBuf.length),
nameBuf,
])
// Encryption record: size, type 1, then its data (dummy here).
const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
}
test.group('rar / RAR 4', () => {
test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => {
const data = Buffer.concat([
RAR4,
rar4Main(0x0001 | 0x0100), // volume, first volume
rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }),
])
assert.equal(rarFormat(data), 'rar4')
const info = parseRarHeaders(data)!
assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true })
assert.deepEqual(listing(info.entries), [
{ name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
])
})
test('detects password-protected files and encrypted headers', ({ assert }) => {
const files = parseRarHeaders(
Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })])
)!
assert.isTrue(files.entries[0].encrypted)
const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))!
assert.isTrue(headers.headersEncrypted)
assert.lengthOf(headers.entries, 0)
})
test('copies out stored entries, not compressed ones', ({ assert }) => {
const data = Buffer.concat([
RAR4,
rar4Main(0),
rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }),
rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }),
])
const [stored, packed] = parseRarHeaders(data)!.entries
data.write('<nzb>', stored.dataOffset)
assert.equal(storedEntryData(data, stored)?.toString(), '<nzb>')
assert.isFalse(packed.stored)
assert.isNull(storedEntryData(data, packed))
})
test('stops at a header with a bad CRC', ({ assert }) => {
const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 })
const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 })
bad[10] ^= 0xff
const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))!
assert.deepEqual(info.entries.map((e) => e.name), ['a.bin'])
})
test('decodes compressed Unicode names', ({ assert }) => {
// "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte,
// then 3 plain bytes), followed by one more flag byte for the last byte.
const ascii = Buffer.from('?.txt', 'latin1')
const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74])
assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt')
assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt')
})
})
test.group('rar / RAR 5', () => {
test('lists files and reads volume flags', ({ assert }) => {
const data = Buffer.concat([
RAR5,
rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume
rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }),
])
assert.equal(rarFormat(data), 'rar5')
const info = parseRarHeaders(data)!
assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false })
assert.deepEqual(listing(info.entries), [
{ name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
])
})
test('a later volume carries a volume number', ({ assert }) => {
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))!
assert.isFalse(info.firstVolume)
})
test('detects encrypted files and encrypted headers', ({ assert }) => {
const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))!
assert.isTrue(files.entries[0].encrypted)
const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))!
assert.isTrue(headers.headersEncrypted)
})
test('marks the listing complete at the end-of-archive header', ({ assert }) => {
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))!
assert.isFalse(info.truncated)
assert.lengthOf(info.entries, 1)
})
test('copies out entries stored without compression', ({ assert }) => {
const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))])
const entry = parseRarHeaders(data)!.entries[0]
assert.isTrue(entry.stored)
data.write('<nzb>', entry.dataOffset)
assert.equal(storedEntryData(data, entry)?.toString(), '<nzb>')
assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry))
assert.isNull(storedEntryData(data, { ...entry, encrypted: true }))
})
test('returns null for other data', ({ assert }) => {
assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1')))
})
})
test.group('rar / release naming from contents', () => {
const info = (names: string[]) =>
({ format: 'rar5', volume: false, firstVolume: true, solid: false, headersEncrypted: false, truncated: true,
entries: names.map((name) => ({ name, size: 1, packedSize: 1, directory: false, encrypted: false, splitBefore: false, splitAfter: false })) }) as any
test('uses the shared top-level directory when it looks like a release name', async ({ assert }) => {
const { releaseNameFromContents } = await import('#services/ArchiveService')
assert.equal(
releaseNameFromContents(info(['GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/00-a.jpg', 'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/01-b.flac'])),
'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM'
)
// No directory: the largest readable entry names the release.
assert.equal(
releaseNameFromContents(info(['Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265.nzb'])),
'Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265'
)
assert.isUndefined(releaseNameFromContents(info(['movie.mkv'])))
assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91PqA7.mkv'])))
assert.isUndefined(releaseNameFromContents(info(['a/x', 'b/y'])))
assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91Pq/x.mkv'])))
})
test('hidden entries (a decoy) name nothing', async ({ assert }) => {
const { releaseNameFromContents, isHidden } = await import('#services/ArchiveService')
const decoy = '.1917-2019-2160p_MA_WEB-DL_Hybrid_H265_DV_HDR_DDP_Atmos_5.1_English-HONE-.mkv'
const listing = info([decoy, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX', `${decoy}/${decoy}.19OMnL`, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX/boston.blue.s01e02.rar'])
listing.entries[0].directory = true
listing.entries[1].directory = true
listing.entries[2].size = 22_093_429_730
assert.equal(releaseNameFromContents(listing), 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX')
assert.isTrue(isHidden(`${decoy}/x.mkv`))
assert.isTrue(isHidden('dir/.hidden.mkv'))
assert.isFalse(isHidden('Some.Release/movie.mkv'))
})
test('recognises obfuscated names', async ({ assert }) => {
const { looksRandom } = await import('#services/ArchiveService')
assert.isTrue(looksRandom('CGBWgdhlSCAOAW62gLWRRM'))
assert.isTrue(looksRandom('0220e975b1417f96b77dc64410e10ebc'))
assert.isFalse(looksRandom('Plus.and.Minus.S01E06.1080p'))
assert.isFalse(looksRandom('1790546808_nicovideo_jp_watch_sm26679715'))
})
})
test.group('rar / looksRandom capitals', () => {
test('long letter-only tokens with many capitals are random', async ({ assert }) => {
const { looksRandom } = await import('#services/ArchiveService')
assert.isTrue(looksRandom('RmISakytFwXSCEzjVEpaKk'))
assert.isTrue(looksRandom('326678d1d07ef12a.bin'))
assert.isFalse(looksRandom('Some.Show.S01E01'))
assert.isFalse(looksRandom('TheQuickBrownFoxJumps'))
})
})
// --- Archive walks (D40) --------------------------------------------------------
/** Volumes split into segments of `segmentSize` bytes; counts the segments read. */
function memoryReader(volumes: Buffer[], segmentSize: number) {
const read = new Set<string>()
const reader: VolumeReader & { segments: Set<string> } = {
segments: read,
volumes: volumes.length,
size: (v) => volumes[v].length,
async read(v, offset, more) {
const index = Math.floor(offset / segmentSize)
if (offset >= volumes[v].length) return null
read.add(`${v}:${index}`)
if (more) read.add(`${v}:${index + 1}`)
return volumes[v].subarray(offset, (index + (more ? 2 : 1)) * segmentSize)
},
}
return reader
}
function rar5Set() {
const main = (n: number) => rar5Block(1, 0, n === 0 ? vint(0x0001) : Buffer.concat([vint(0x0003), vint(n)]))
const end = rar5Block(5, 0, vint(0x0001))
const SPLIT_BEFORE = 0x0008
const SPLIT_AFTER = 0x0010
return [
Buffer.concat([RAR5, main(0), rar5File('Book/01.mp3', 3000, { packed: 3000 }), rar5File('Book/02.mp3', 7000, { packed: 5000, headerFlags: SPLIT_AFTER }), end]),
Buffer.concat([
RAR5,
main(1),
rar5File('Book/02.mp3', 7000, { packed: 2000, headerFlags: SPLIT_BEFORE }),
rar5File('Book/03.mp3', 1000, { packed: 1000 }),
rar5File('Book/04.mp3', 4500, { packed: 4000, headerFlags: SPLIT_AFTER }),
end,
]),
Buffer.concat([RAR5, main(2), rar5File('Book/04.mp3', 4500, { packed: 500, headerFlags: SPLIT_BEFORE }), rar5File('Book/05.mp3', 100, { packed: 100 }), rar5Block(5, 0, vint(0))]),
]
}
test.group('rar / archive walks', () => {
test('lists every entry of a RAR5 set, reading a segment per header', async ({ assert }) => {
// 700-byte segments: headers get cut off by segment ends.
const reader = memoryReader(rar5Set(), 700)
const walk = await walkRar(reader)
assert.isTrue(walk.complete)
assert.equal(walk.format, 'rar5')
assert.deepEqual(walk.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3', 'Book/03.mp3', 'Book/04.mp3', 'Book/05.mp3'])
assert.equal(walk.entries[1].size, 7000)
const total = rar5Set().reduce((sum, v) => sum + Math.ceil(v.length / 700), 0)
assert.isBelow(reader.segments.size, total / 2)
})
test('lists a RAR4 set across volumes', async ({ assert }) => {
const end = rar4Block(0x7b, 0, Buffer.alloc(0))
const volumes = [
Buffer.concat([RAR4, rar4Main(0x0001 | 0x0100), rar4File(Buffer.from('a.flac'), { packed: 900, unpacked: 900 }), rar4File(Buffer.from('b.flac'), { packed: 2000, unpacked: 2500, flags: 0x0002 }), end]),
Buffer.concat([RAR4, rar4Main(0x0001), rar4File(Buffer.from('b.flac'), { packed: 500, unpacked: 2500, flags: 0x0001 }), rar4File(Buffer.from('c.flac'), { packed: 10, unpacked: 10 }), end]),
]
const walk = await walkRar(memoryReader(volumes, 512))
assert.isTrue(walk.complete)
assert.deepEqual(walk.entries.map((e) => e.name), ['a.flac', 'b.flac', 'c.flac'])
})
test('stops at a missing segment or a broken header, keeping what it found', async ({ assert }) => {
const set = rar5Set()
const missing = memoryReader(set, 700)
const read = missing.read.bind(missing)
missing.read = async (v, offset, more) => (v === 1 ? null : read(v, offset, more))
const partial = await walkRar(missing)
assert.equal(partial.stopped, 'unreadable')
assert.deepEqual(partial.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3'])
const broken = rar5Set()
// The header of 03.mp3 in volume 2, after 02.mp3's continuation and its data.
const at = broken[1].indexOf(Buffer.from('Book/03.mp3')) - 4
broken[1][at] ^= 0xff
const invalid = await walkRar(memoryReader(broken, 700))
assert.equal(invalid.stopped, 'invalid')
assert.lengthOf(invalid.entries, 2)
})
test('finds segments from their yEnc offsets', async ({ assert }) => {
const { SegmentReader } = await import('#lib/rar/index')
const set = rar5Set()
const size = 700
const articles = new Map<string, Buffer>()
const files = set.map((volume, v) => {
const segments = []
for (let i = 0; i * size < volume.length; i++) {
const id = `v${v}s${i}@test`
articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
segments.push({ number: i + 1, id, size: 0 })
}
return segments
})
const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
const walk = await walkRar(reader)
assert.isTrue(walk.complete)
assert.lengthOf(walk.entries, 5)
assert.isBelow(reader.fetched, articles.size / 2)
const capped = new SegmentReader(files, 3, async (id) => articles.get(id)!)
const stopped = await walkRar(capped)
assert.equal(stopped.stopped, 'unreadable')
assert.equal(capped.fetched, 3)
})
})
/** One yEnc article of `file`: part `index` of `size` bytes. */
function yencArticle(name: string, file: Buffer, index: number, size: number) {
const data = file.subarray(index * size, (index + 1) * size)
const total = Math.ceil(file.length / size)
const out: number[] = []
for (const byte of data) {
const c = (byte + 42) & 0xff
// Escape the critical characters, and '.' and whitespace so no line needs dot-stuffing.
if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
else out.push(c)
}
const lines: string[] = []
const encoded = Buffer.from(out)
for (let i = 0; i < encoded.length; i += 128) {
let end = Math.min(encoded.length, i + 128)
if (encoded[end - 1] === 0x3d && end < encoded.length) end++
lines.push(encoded.subarray(i, end).toString('latin1'))
i = end - 128
}
const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
return Buffer.from(head + lines.join('\r\n') + tail, 'latin1')
}
tests/unit/release.spec.ts
-169
View File
@@ -1,169 +0,0 @@
import { test } from '@japa/runner'
import { fileStem, releaseIdentity, subjectRemainder } from '#lib/release/index'
import { looksLikeFilename, weakFileKey } from '#lib/subject/index'
test.group('release / fileStem', () => {
test('reduces all files of a set to the same stem', ({ assert }) => {
const names = [
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst',
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.par2',
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.vol03+04.par2',
]
assert.deepEqual(new Set(names.map(fileStem)), new Set(['1790546808_nicovideo_jp_watch_sm26679715']))
})
test('handles rar, split and par2 volume naming', ({ assert }) => {
assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0')
assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024')
assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024')
assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265')
assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264')
assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024')
assert.equal(fileStem('Some.Movie.2024.nfo'), 'Some.Movie.2024')
assert.equal(fileStem('00-artist-album-web-2026.m3u'), '00-artist-album-web-2026')
assert.equal(fileStem('f6dd6b156692a52b.bin'), 'f6dd6b156692a52b')
})
test('never reduces a name to nothing', ({ assert }) => {
assert.equal(fileStem('.par2'), '.par2')
})
})
test.group('release / subjectRemainder', () => {
test('is empty when the subject only has counters, filename and size', ({ assert }) => {
assert.equal(subjectRemainder('[1/4] - "x.tar.zst" yEnc 15411409'), '')
assert.equal(subjectRemainder('. - [00/88] - "60SHkTQSvn5RZGExwju2.nzb" yEnc'), '')
})
test('keeps a release name written outside the quotes', ({ assert }) => {
assert.equal(
subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'),
'Some Show S01E02 1080p'
)
})
})
test.group('release / releaseIdentity', () => {
test('groups a par2 set with its archive by stem, per poster', ({ assert }) => {
const base = { poster: 'p', weak: false }
const a = releaseIdentity({ ...base, subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
const b = releaseIdentity({ ...base, subject: '[3/4] - "x.tar.zst.vol01+02.par2" yEnc 2', name: 'x.tar.zst.vol01+02.par2' })!
assert.equal(a.key, b.key)
assert.equal(a.name, 'x')
assert.equal(a.fileTotal, 4)
const other = releaseIdentity({ ...base, poster: 'q', subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
assert.notEqual(a.key, other.key)
})
test('a par2 index posted without [i/M] counter joins its release', ({ assert }) => {
const base = { poster: 'nicovideo_archiver', weak: false }
const archive = releaseIdentity({ ...base, subject: '[1/6] - "x.tar.zst" yEnc 105054576', name: 'x.tar.zst' })!
const index = releaseIdentity({ ...base, subject: '"x.tar.zst.par2" yEnc 3100', name: 'x.tar.zst.par2' })!
assert.equal(index.key, archive.key)
assert.equal(archive.fileTotal, 6)
assert.isUndefined(index.fileTotal)
})
test('uses the subject text when it names the release', ({ assert }) => {
const base = { poster: 'p', weak: false }
const a = releaseIdentity({ ...base, subject: 'My.Release [01/40] - "abc.part01.rar" yEnc', name: 'abc.part01.rar' })!
const b = releaseIdentity({ ...base, subject: 'My.Release [40/40] - "abc.par2" yEnc', name: 'abc.par2' })!
assert.equal(a.key, b.key)
assert.equal(a.name, 'My Release')
})
test('obfuscated volumes group by stem regardless of the random poster', ({ assert }) => {
const a = releaseIdentity({ poster: '3Uzx7TTSFpLzz@ngPost.com', subject: '"lJzu.part033.rar" yEnc', name: 'lJzu.part033.rar', weak: true })!
const b = releaseIdentity({ poster: 'bGLd1oaXRdrK1@ngPost.com', subject: '"lJzu.part034.rar" yEnc', name: 'lJzu.part034.rar', weak: true })!
assert.equal(a.key, b.key)
})
test('random extensionless names cannot be grouped by name', ({ assert }) => {
const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'
assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true }))
})
})
test.group('subject / weakFileKey', () => {
test('ignores random per-article names, keeps real filenames', ({ assert }) => {
assert.isFalse(looksLikeFilename('204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'))
assert.isTrue(looksLikeFilename('lJzuoIKA3T8xjzAD3vqtw0.part033.rar'))
// Same file, different random names per article: same key.
assert.equal(weakFileKey('aaaa1111', 5000, 7), weakFileKey('bbbb2222', 5000, 7))
// Equal-sized volumes with real names: different keys.
assert.notEqual(weakFileKey('x.part01.rar', 5000, 7), weakFileKey('x.part02.rar', 5000, 7))
assert.notEqual(weakFileKey('aaaa1111', 5000, 7), weakFileKey('aaaa1111', 5001, 7))
})
})
test.group('release / rarReleaseIdentity', () => {
test('groups volumes by the shared top-level directory', async ({ assert }) => {
const { rarReleaseIdentity } = await import('#lib/release/index')
const first = rarReleaseIdentity([
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/00-cover.jpg', size: 589382 },
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/01-track.flac', size: 30_000_000 },
])!
const later = rarReleaseIdentity([{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/03-track.flac', size: 29_000_000 }])!
assert.equal(first.key, later.key)
assert.equal(first.name, 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM')
assert.equal(first.source, 'rar')
})
test('without a directory, volumes of one file share its first entry', async ({ assert }) => {
const { rarReleaseIdentity } = await import('#lib/release/index')
const a = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
const b = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
assert.equal(a.key, b.key)
assert.equal(a.name, 'Some.Show.S01E01.1080p')
assert.isNull(rarReleaseIdentity([]))
})
})
test.group('release / namesRelease', () => {
test('random per-file tags and poster ids are not a release name', async ({ assert }) => {
const { namesRelease, subjectRemainder, releaseIdentity } = await import('#lib/release/index')
const subject = 'elys-de81bcabef5faf6a - [Nyh5Q07Y] "JkpNIFg7En50zU1U" yEnc'
assert.isFalse(namesRelease(subjectRemainder(subject)))
const a = releaseIdentity({ poster: 'elys', subject, name: 'JkpNIFg7En50zU1U', weak: false })!
assert.equal(a.source, 'stem')
assert.isTrue(namesRelease(subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc')))
assert.isTrue(namesRelease('My Release'))
})
})
test.group('release / post id', () => {
test('groups random-named files of one post by poster, post id and file count', async ({ assert }) => {
const { releaseIdentity, isPostToken } = await import('#lib/release/index')
const base = { poster: 'jW344UtC8R2bZEegrVLz@x', weak: false }
const a = releaseIdentity({ ...base, subject: '83110629-n [10/18] - "eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k" yEnc', name: 'eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k' })!
const b = releaseIdentity({ ...base, subject: '83110629-n [01/18] - "2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0" yEnc', name: '2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0' })!
assert.equal(a.key, b.key)
assert.equal(a.source, 'post')
assert.equal(a.fileTotal, 18)
const other = releaseIdentity({ ...base, subject: '83110630-n [01/18] - "x" yEnc', name: 'x' })!
assert.notEqual(a.key, other.key)
assert.isTrue(isPostToken('83110629 n'))
assert.isFalse(isPostToken('PRiVATE'))
assert.isFalse(isPostToken(''))
// Without a counter there is nothing to bound the post: fall back to the stem.
assert.equal(releaseIdentity({ ...base, subject: '83110629-n - "x.bin" yEnc', name: 'x.bin' })!.source, 'stem')
})
})
test.group('release / par2 names', () => {
test('names a season pack after what its episodes share', async ({ assert }) => {
const { par2ReleaseName } = await import('#lib/release/index')
const eps = Array.from({ length: 20 }, (_, i) =>
`SEAL.Team.S03E${String(i + 1).padStart(2, '0')}.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato.mkv`)
assert.equal(par2ReleaseName(eps), 'SEAL.Team.S03.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato')
// Movie plus sample and nfo: the movie's (longest) name.
assert.equal(
par2ReleaseName(['Some.Movie.2024.1080p.BluRay.x264-GRP.mkv', 'Some.Movie.2024.1080p.BluRay.x264-GRP.nfo', 'sample.mkv']),
'Some.Movie.2024.1080p.BluRay.x264-GRP'
)
// Numbered parts: the shared name without the part number.
assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP')
assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024')
})
})
tests/unit/seed_fetch_pool.spec.ts
-48
View File
@@ -1,48 +0,0 @@
import { test } from '@japa/runner'
import net from 'node:net'
import { unzipSync } from 'node:zlib'
import nntpConfig from '#config/nntp'
import indexerConfig from '#config/indexer'
import SeedFetchPool from '#services/SeedFetchPool'
import { parseOverviews } from '#lib/nntp/xover_client'
test.group('SeedFetchPool', () => {
test('a chunk that fails (dead connection, stalled compressed reply) is fetched again uncompressed', async ({ assert }) => {
let connections = 0
const compressAsked: Record<number, boolean> = {}
const server = net.createServer((sock) => {
const n = ++connections
sock.write('200 welcome\r\n')
sock.on('data', (chunk) => {
for (const line of chunk.toString().split('\r\n').filter(Boolean)) {
if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
else if (line.startsWith('XFEATURE')) (compressAsked[n] = true), sock.write('500 no\r\n')
else if (line.startsWith('GROUP')) sock.write('211 1 7 7 a.b.test\r\n')
// The first connection dies as the request arrives, like one the server dropped while idle.
else if (line.startsWith('XOVER') && n === 1) sock.destroy()
else if (line.startsWith('XOVER')) sock.write('224 overviews\r\n7\tsubject\tposter\tdate\t<a@b>\t\t10\t1\r\n.\r\n')
}
})
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const saved = { host: nntpConfig.host, port: nntpConfig.port, secure: nntpConfig.secure, connections: indexerConfig.seed.nntpConnections, compress: indexerConfig.seed.nntpCompress }
// The fallback from a compressed pooled connection (the default).
indexerConfig.seed.nntpCompress = true
Object.assign(nntpConfig, { host: '127.0.0.1', port: (server.address() as net.AddressInfo).port, secure: false })
indexerConfig.seed.nntpConnections = 2
try {
const raw = await SeedFetchPool.fetch('a.b.test', 7, 7)
assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [7])
assert.equal(connections, 2)
// The pooled connection asked for gzip; the retry didn't.
assert.deepEqual(compressAsked, { 1: true })
} finally {
SeedFetchPool.closeIdle()
Object.assign(nntpConfig, { host: saved.host, port: saved.port, secure: saved.secure })
indexerConfig.seed.nntpConnections = saved.connections
indexerConfig.seed.nntpCompress = saved.compress
server.close()
}
})
})
tests/unit/seed_plan.spec.ts
-16
View File
@@ -1,16 +0,0 @@
import { test } from '@japa/runner'
import { cut, uncovered } from '#lib/seed/plan'
test.group('seed plan', () => {
test('plans only what earlier ranges leave uncovered', ({ assert }) => {
// a.b.ath on 2026-10-06: test ranges around 49.8B and a plan below them; the backfill ends at 58.4B.
assert.deepEqual(uncovered(1, 100, [[40, 50], [10, 20], [45, 60]]), [[1, 9], [21, 39], [61, 100]])
assert.deepEqual(uncovered(1, 100, []), [[1, 100]])
assert.deepEqual(uncovered(1, 100, [[0, 200]]), [])
assert.deepEqual(uncovered(10, 20, [[1, 5], [25, 30]]), [[10, 20]])
})
test('cuts gaps into ranges, newest first', ({ assert }) => {
assert.deepEqual(cut([[1, 9], [61, 100]], 25), [[76, 100], [61, 75], [1, 9]])
})
})
tests/unit/sevenzip.spec.ts
-95
View File
@@ -1,95 +0,0 @@
import { test } from '@japa/runner'
import {
checkHeader,
encodedHeaderEncrypted,
listingStart,
parseBsdtarListing,
parseSevenZipStart,
readNumber,
sevenZipVolume,
} from '#lib/sevenzip/index'
// `7z a tiny.7z Tiny.Release` (Tiny.Release/a.txt): an LZMA-encoded header.
const ENCODED = Buffer.from(
'N3q8ryccAATeIVLCfQAAAAAAAAAhAAAAAAAAAGkLW/MBAAtoZWxsbyB3b3JsZAoAAACBMweuD87ysgwHyEN/QbH6/duIttdfpOXRhZbi6t7tycGOUDyW1l2aTE7xmuZ/NAXKgg599VxKYAF1XImsfT9LzHQKaEH6PEZambLnwJM62mXXEJIpaYvhCrHFeQv+jLnYjgJMfwZEcAAAABcGEAEJbQAHCwEAASMDAQEFXQAQAAAMgJYKASTou9wAAA==',
'base64'
)
// The same with -mhc=off: a plain header.
const PLAIN = Buffer.from(
'N3q8ryccAASpRwRsEAAAAAAAAACWAAAAAAAAACTou9wBAAtoZWxsbyB3b3JsZAoAAQQGAAEJEAAHCwEAASEhAQAMDAAICgEtOwivAAAFAg4BgBkJAAAAAAAAAAAAEUEAVABpAG4AeQAuAFIAZQBsAGUAYQBzAGUAAABUAGkAbgB5AC4AUgBlAGwAZQBhAHMAZQAvAGEALgB0AHgAdAAAABkCAAAUEgEA/QYYLBBR3QH9BhgsEFHdARUKAQAQgP1BIIC0gQAA',
'base64'
)
// `7z a -pSecret -mhe=on`: an encrypted header; with -mhe=off only the file data is encrypted.
const HEADER_ENCRYPTED = Buffer.from(
'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
'base64'
)
const DATA_ENCRYPTED = Buffer.from(
'N3q8ryccAASBXGg3kQAAAAAAAAAiAAAAAAAAAO4xlWU16N8rTbvoF1dbq3eLPdZiAACBMweuD87ysgwHsMPa919FiphISvi++OI2giqcUPCAdmXzZtkqFSqP0Nr4rWaRH5dR+vNA2jJGIbDAZpbQM/ka9oh27tkqKb/+dHyx1SFHkxbLyWvd7W0qwaDHxSvW2fTpRdJSmi74crF957Qwy+P8dDF0MOYr5AGHAQHJSAAAFwYQAQmAgQAHCwEAASMDAQEFXQAQAAAMgKYKAaUZQIYAAA==',
'base64'
)
const headerOf = (archive: Buffer) => {
const start = parseSevenZipStart(archive)!
return archive.subarray(start.headerStart, start.archiveSize)
}
test.group('sevenzip', () => {
test('tells encrypted headers from the encoded header alone', ({ assert }) => {
assert.isTrue(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED)))
assert.isFalse(encodedHeaderEncrypted(headerOf(DATA_ENCRYPTED)))
assert.isFalse(encodedHeaderEncrypted(headerOf(ENCODED)))
// A plain header isn't encoded: bsdtar lists it.
assert.isNull(encodedHeaderEncrypted(headerOf(PLAIN)))
assert.isNull(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED).subarray(0, 12)))
})
test('finds the header at the end from the signature header', ({ assert }) => {
const start = parseSevenZipStart(ENCODED)!
assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })
const header = ENCODED.subarray(start.headerStart, start.archiveSize)
assert.isTrue(checkHeader(start, header))
assert.isFalse(checkHeader(start, Buffer.alloc(33)))
// Encoded: the packed header right after the file data (packPos 16) is needed too.
assert.equal(listingStart(start, header), 48)
const plain = parseSevenZipStart(PLAIN)!
assert.equal(plain.archiveSize, PLAIN.length)
assert.equal(listingStart(plain, PLAIN.subarray(plain.headerStart)), plain.headerStart)
})
test('rejects other data and a damaged signature header', ({ assert }) => {
assert.isNull(parseSevenZipStart(Buffer.from('Rar!\x1a\x07\x01\x00'.padEnd(40, '\0'), 'latin1')))
const damaged = Buffer.from(ENCODED)
damaged[14] ^= 1
assert.isNull(parseSevenZipStart(damaged))
})
test('reads 7z numbers', ({ assert }) => {
assert.deepEqual(readNumber(Buffer.from([0x10]), 0), [16, 1])
assert.deepEqual(readNumber(Buffer.from([0x81, 0x02]), 0), [0x102, 2])
assert.deepEqual(readNumber(Buffer.from([0xc0, 0x34, 0x12]), 0), [0x1234, 3])
assert.deepEqual(readNumber(Buffer.from([0xff, 1, 0, 0, 0, 0, 0, 0, 0]), 0), [1, 9])
})
test('parses bsdtar listings', ({ assert }) => {
const out = [
'drwxrwxr-x 0 0 0 0 Sep 30 19:13 Some.Release.2026/',
'-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release.2026/movie.mkv',
'-rw-rw-r-- 0 0 0 3 Jan 1 2019 Some.Release.2026/some info.nfo',
'',
].join('\n')
assert.deepEqual(parseBsdtarListing(out), [
{ name: 'Some.Release.2026', size: 0, directory: true },
{ name: 'Some.Release.2026/movie.mkv', size: 3000000, directory: false },
{ name: 'Some.Release.2026/some info.nfo', size: 3, directory: false },
])
})
test('numbers volumes', ({ assert }) => {
assert.equal(sevenZipVolume('x.7z.001'), 1)
assert.equal(sevenZipVolume('X.7Z.012'), 12)
assert.equal(sevenZipVolume('x.7z'), 0)
assert.isNull(sevenZipVolume('x.part01.rar'))
})
})
tests/unit/split.spec.ts
-162
View File
@@ -1,162 +0,0 @@
import { test } from '@japa/runner'
import { splitInstances, splitVerified, verifyVolume, type SplitArticle } from '#lib/split/index'
import { crc32Hex } from '#lib/yenc/index'
/** Deterministic pseudo-random numbers. */
function rng(seed: number) {
let x = seed
return () => {
x = (x * 1103515245 + 12345) & 0x7fffffff
return x / 0x7fffffff
}
}
interface PostOptions {
volumes: number
parts: number
/** Parallel upload connections; they finish slightly out of order. */
connections?: number
/** First article number of the post. */
start?: number
/** Unrelated articles between two articles of this post (other posters). */
noise?: number
/** Fraction of articles that never arrive. */
drop?: number
label?: string
seed?: number
}
/** Simulates a posting tool uploading equal-sized volumes. */
function post(opts: PostOptions): SplitArticle[] {
const random = rng(opts.seed ?? 1)
const connections = opts.connections ?? 1
const label = opts.label ?? 'p'
const queue: { volume: number; number: number }[] = []
for (let v = 0; v < opts.volumes; v++) for (let n = 1; n <= opts.parts; n++) queue.push({ volume: v, number: n })
const articles: SplitArticle[] = []
let articleNumber = opts.start ?? 1000
for (let i = 0; i < queue.length; i += connections) {
// One round: each connection posts one article; they land in random order.
const round = queue.slice(i, i + connections).sort(() => random() - 0.5)
for (const { volume, number } of round) {
articleNumber += 1 + Math.floor(random() * (opts.noise ?? 0))
if (random() < (opts.drop ?? 0)) continue
articles.push({ number, messageId: `${label}${volume}.${number}`, bytes: 700_000, articleNumber })
}
}
return articles
}
/** Source volume of an article, from its simulated message-id. */
const source = (a: SplitArticle) => a.messageId.split('.')[0]
/** Every result volume must contain articles of one source volume only. */
function assertPure(assert: any, volumes: SplitArticle[][]) {
for (const volume of volumes) {
assert.lengthOf(new Set(volume.map(source)), 1, `mixed volume: ${volume.map((a) => a.messageId).join(' ')}`)
}
}
test.group('split / splitInstances', () => {
test('separates sequentially posted volumes', ({ assert }) => {
const volumes = splitInstances(post({ volumes: 3, parts: 14 }), { totalParts: 14 })
assert.lengthOf(volumes, 3)
assertPure(assert, volumes)
assert.isTrue(volumes.every((v) => v.length === 14))
})
test('separates volumes posted over interleaved connections, out of order', ({ assert }) => {
for (const seed of [1, 2, 3, 4, 5]) {
const volumes = splitInstances(post({ volumes: 6, parts: 40, connections: 8, noise: 30, seed }), { totalParts: 40 })
assertPure(assert, volumes)
assert.lengthOf(volumes, 6, `seed ${seed}`)
}
})
test('a volume that completes is never mixed, even with heavy loss', ({ assert }) => {
for (const seed of [11, 12, 13, 14, 15, 16, 17, 18]) {
const articles = post({ volumes: 5, parts: 30, connections: 6, noise: 20, drop: 0.2, seed })
const volumes = splitInstances(articles, { totalParts: 30 })
assertPure(assert, volumes.filter((v) => v.length === 30))
}
})
test('a missing article only affects its own volume', ({ assert }) => {
const articles = post({ volumes: 4, parts: 20, connections: 4, drop: 0.05, seed: 7 })
const volumes = splitInstances(articles, { totalParts: 20 })
assertPure(assert, volumes)
assert.lengthOf(volumes, 4)
})
test('posts of the same size at different times do not mix', ({ assert }) => {
// An earlier post that lost articles (e.g. before the backfill window), then two more.
const articles = [
...post({ volumes: 3, parts: 14, connections: 4, drop: 0.3, label: 'a', start: 1_000, seed: 3 }),
...post({ volumes: 3, parts: 14, connections: 4, label: 'b', start: 200_000, seed: 4 }),
...post({ volumes: 2, parts: 14, connections: 2, label: 'c', start: 900_000, seed: 5 }),
]
const volumes = splitInstances(articles, { totalParts: 14 })
// The damaged post may produce mixed volumes, but those stay incomplete
// (they never become files); every volume that completes is pure.
const complete = volumes.filter((v) => v.length === 14)
assertPure(assert, complete)
assert.includeMembers(complete.map((v) => source(v[0])), ['b0', 'b1', 'b2', 'c0', 'c1'])
// The later, intact posts are not affected at all.
assertPure(assert, volumes.filter((v) => !source(v[0]).startsWith('a')))
})
test('ignores duplicate message-ids and does not depend on input order', ({ assert }) => {
const articles = post({ volumes: 2, parts: 5, connections: 3, seed: 9 })
const shuffled = [...articles].reverse().concat(articles[0])
assert.deepEqual(splitInstances(shuffled, { totalParts: 5 }), splitInstances(articles, { totalParts: 5 }))
})
})
/** Like post(), with real data per volume so part and file CRCs are genuine. */
function postWithCrcs(opts: PostOptions): SplitArticle[] {
const random = rng((opts.seed ?? 1) + 1000)
const size = 64
const data = new Map<string, Buffer>()
const articles = post(opts)
for (const a of articles) {
const volume = source(a)
if (!data.has(volume)) {
data.set(volume, Buffer.from(Array.from({ length: opts.parts * size }, () => Math.floor(random() * 256))))
}
const file = data.get(volume)!
a.pcrc32 = crc32Hex(file.subarray((a.number - 1) * size, a.number * size))
a.partSize = size
a.fileCrc32 = a.number === opts.parts ? crc32Hex(file) : null
}
return articles
}
test.group('split / verified', () => {
test('verifyVolume accepts a correct volume and rejects a mixed one', ({ assert }) => {
const articles = postWithCrcs({ volumes: 2, parts: 6 })
const volumeA = articles.filter((a) => source(a) === 'p0')
assert.isTrue(verifyVolume(volumeA, 6))
const mixed = volumeA.map((a) => (a.number === 3 ? articles.find((b) => source(b) === 'p1' && b.number === 3)! : a))
assert.isFalse(verifyVolume(mixed, 6))
assert.isFalse(verifyVolume(volumeA.slice(0, 5), 6))
})
test('only verified volumes come out, and they are never mixed', ({ assert }) => {
for (const seed of [21, 22, 23, 24, 25, 26]) {
const articles = [
...postWithCrcs({ volumes: 4, parts: 30, connections: 16, noise: 20, drop: 0.05, label: 'a', seed }),
...postWithCrcs({ volumes: 3, parts: 30, connections: 16, noise: 20, label: 'b', start: 300_000, seed: seed + 50 }),
]
for (const volume of splitVerified(articles, { totalParts: 30 })) {
assert.isTrue(verifyVolume(volume, 30))
assertPure(assert, [volume])
}
}
})
test('recovers every volume of a cleanly posted set', ({ assert }) => {
const volumes = splitVerified(postWithCrcs({ volumes: 5, parts: 20, connections: 4, noise: 10, seed: 31 }), { totalParts: 20 })
assert.lengthOf(volumes, 5)
})
})
tests/unit/spotnet.spec.ts
-186
View File
@@ -1,186 +0,0 @@
import { test } from '@japa/runner'
import { deflateRawSync } from 'node:zlib'
import {
decodeSpotBinary,
decodeSpotText,
parseSpotArticleHeaders,
parseSpotHeader,
parseSpotXml,
spotCategoryLabels,
verifySpotHeader,
} from '#lib/spotnet/index'
import { ubbToText } from '#lib/newznab/xml'
// A real spot from free.pt (self-signed, key 7), 2026-09-28.
const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "<jWCbaAa4f1A7LC6agmOEx@spot.net>", "date": "Mon, 28 Sep 2026 18:24:45 GMT"}
test.group('spotnet / headers', () => {
test('parses a spot header', ({ assert }) => {
const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)!
assert.include(spot, {
poster: 'Bart',
title: 'The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi',
tag: 'Bart',
category: 0,
keyId: 7,
size: 21651233204,
messageId: 'jWCbaAa4f1A7LC6agmOEx@spot.net',
})
assert.deepEqual(spot.subcats, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] })
})
test('verifies its signatures, and rejects another message id', ({ assert }) => {
const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)!
assert.isTrue(verifySpotHeader(spot))
assert.isFalse(verifySpotHeader({ ...spot, messageId: 'other@spot.net' }))
assert.isNull(parseSpotHeader('someone <x@y>', 'subject', '<a@b>'))
})
})
test.group('spotnet / xml and nzb', () => {
test('reads the NZB segments and category from the X-XML lines', ({ assert }) => {
const raw = [
'From: Bart <x@y>',
'X-XML: <Spotnet><Posting><Key>7</Key><Title>A &amp; B</Title><Category>01<Sub>01a09</Sub>',
'X-XML: <Sub>01z00</Sub></Category><NZB><Segment>abc@spot.net</Segment><Segment>def@spot.net</Segment></NZB>',
'X-XML: <Size>123</Size></Posting></Spotnet>',
].join('\r\n')
const { headers, xml } = parseSpotArticleHeaders(raw)
assert.equal(headers.from, 'Bart <x@y>')
const spot = parseSpotXml(xml)!
assert.deepEqual(spot.nzbSegments, ['abc@spot.net', 'def@spot.net'])
assert.include(spot, { title: 'A & B', category: 0, size: 123 })
assert.deepEqual(spot.subcats, ['01a09', '01z00'])
})
test('reads the description, tag and image', ({ assert }) => {
const spot = parseSpotXml(
`<Spotnet><Posting><Key>7</Key><Tag>Bart</Tag><Title>T</Title>` +
`<Description>Lily Tomlin[br][br]If you&#39;re &#8216;in&#8217; [url=https://example.com/a]here[/url]</Description>` +
`<Image Width='185' Height='278'><Segment>FYe6l94kQsY7LC6agh5Xn@spot.net</Segment></Image>` +
`<Category>01<Sub>01a09</Sub></Category></Posting></Spotnet>`
)!
assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]')
assert.equal(spot.tag, 'Bart')
assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 })
// Older spots link an image on the web.
assert.deepEqual(parseSpotXml('<Posting><Image>http://x.nl/a.jpg</Image></Posting>')!.image, { url: 'http://x.nl/a.jpg' })
assert.isNull(parseSpotXml('<Posting><Title>T</Title></Posting>')!.image)
})
test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => {
const raw = [
'From: vanker',
' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>',
'Subject: The postman | The postman |',
'X-XML: <SpotNet><Posting><Title>The postman</Title><Description><![CDATA[post voor de lol.<br />Van &quot;The',
'X-XML: postman&quot;]]></Description><Image>http://covertarget.com/si.php?id=1</Image><Category>00</Category></Posting></SpotNet>',
'Date: 01 Feb 2010 02:43:16 GMT',
].join('\r\n')
const { headers, xml } = parseSpotArticleHeaders(raw)
assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>')
assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT')
const spot = parseSpotXml(xml)!
assert.equal(spot.description, 'post voor de lol.<br />Van "The postman"')
assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' })
assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"')
})
test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => {
assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café')
assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café')
})
test('names categories as Spotweb does', ({ assert }) => {
assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), {
head: 'Image',
subcats: [
{ type: 'Type', values: ['Movie'] },
{ type: 'Format', values: ['x264'] },
{ type: 'Source', values: ['Retail'] },
{ type: 'Language', values: ['English subtitles (available)', 'English audio/written'] },
{ type: 'Genre', values: ['Science Fiction'] },
],
})
assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] })
})
test('turns UBB markup into plain text', ({ assert }) => {
assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)')
})
test('decodes an escaped, deflated NZB split over articles', ({ assert }) => {
const nzb = '<?xml version="1.0"?>\n<nzb>\r\n=\0</nzb>'.repeat(20)
const escaped = deflateRawSync(Buffer.from(nzb, 'latin1'))
.toString('latin1')
.replace(/[=\n\r\0]/g, (c) => ({ '=': '=D', '\n': '=C', '\r': '=B', '\0': '=A' })[c]!)
// Posted as lines, dot-stuffed, over two articles.
const lines = escaped.match(/.{1,40}/gs)!.map((l) => (l.startsWith('.') ? '.' + l : l))
const half = Math.ceil(lines.length / 2)
const bodies = [lines.slice(0, half), lines.slice(half)].map((ls) => Buffer.from(ls.join('\r\n'), 'latin1'))
assert.equal(decodeSpotBinary(bodies).toString('latin1'), nzb)
})
})
test.group('spotnet / moderation', () => {
test('recognises delete and dispose commands from key 2 only', async ({ assert }) => {
const { moderationTarget, spotterId } = await import('#lib/spotnet/index')
assert.equal(moderationTarget({ keyId: 2, title: 'DISPOSE 7IzfA0aCFTkOne6agvY06@spot.net - The Voice DE S15E07 (1080p)' }), '7IzfA0aCFTkOne6agvY06@spot.net')
assert.equal(moderationTarget({ keyId: 2, title: 'delete <abc@spot.net>' }), 'abc@spot.net')
assert.isNull(moderationTarget({ keyId: 7, title: 'DISPOSE abc@spot.net' }))
assert.isNull(moderationTarget({ keyId: 2, title: 'The Disposal 2024' }))
// Spotweb: base64 of the little-endian CRC32 of the key, without / + =.
assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/)
})
})
test.group('spotnet / passwords in descriptions', () => {
test('finds passwords written in the description or title', async ({ assert }) => {
const { passwordFromSpot } = await import('#lib/spotnet/index')
assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2')
assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R')
assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813')
assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401')
assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo')
assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t')
assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
})
test('finds passwords in the phrasings found in spots that the first version missed', async ({ assert }) => {
const { passwordFromSpot } = await import('#lib/spotnet/index')
// LOG.md 2026-10-08 (10)
assert.equal(passwordFromSpot('T', 'Formaat: DVD5 ISO[br][br]!! LETOP Wachtwoord : oscfyajg !!'), 'oscfyajg')
assert.equal(passwordFromSpot('T', "Het totaal gewicht komt neer op zo'n 400GB. Deze is voorzien van een wachtwoord. Deze is: VTSm7nLdZZl5Kren[br]"), 'VTSm7nLdZZl5Kren')
assert.equal(passwordFromSpot('T', 'Futurama S11E10 1080P[br][br]ww in nzb: cI6f7dBV6BKkhqD[br][br]Dit was'), 'cI6f7dBV6BKkhqD')
assert.equal(passwordFromSpot('T', '[b]Het wachtwoord voor de nzb is: dw0022[/b][br]'), 'dw0022')
assert.equal(passwordFromSpot('T', 'haakinstructies.[br][br]Wachtwoord inbegrepen: wysiwyg'), 'wysiwyg')
assert.equal(passwordFromSpot('T', 'afgeslagen moeten worden.[br][br]wachtwoord is arn2'), 'arn2')
assert.equal(passwordFromSpot('T', 'Forced : No[br][br]WW:[br]tijdloos[br][br]Zag en merkte'), 'tijdloos')
assert.equal(passwordFromSpot('T', 'telekinetische krachten beschikt.[br]WW carrie[br][br]Multisubs'), 'carrie')
})
test('ignores sentences about passwords', async ({ assert }) => {
const { passwordFromSpot } = await import('#lib/spotnet/index')
for (const description of [
'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]',
'[b]--> Geen wachtwoord benodigd [/b]',
'zitten 2 bestanden in en is met een password beschermd.',
'It also removes the need for manual password entry and improves the reliability',
'Codeermodus: Hoge kwaliteit (2-Pass VBR)',
'29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper',
'Wachtwoord: geen',
'WW: zie nzb',
'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
'Mijn wachtwoord is hier direct te vinden, je moet het wel zelf overtypen',
'Bringing History to Life - Greatest Battles of WW II, Sept 2026',
'WW II',
'WW Disc 1 = 5YIMKDaoTaKe8WIWEKUF',
'U kunt eigenaars- en gebruikerswachtwoorden aan uw pdf toevoegen met behulp van 256-bits encryptie.',
'APP MISC - SHAREit v6.27.49 ww 4062749 arm64 v8a.apk',
'Het ww staat op het plaatje, type het EXACT zo in.',
'Wachtwoord:[br]Deze download heeft er geen',
]) {
assert.isNull(passwordFromSpot('T', description), description)
}
})
})
tests/unit/subject.spec.ts
-108
View File
@@ -1,108 +0,0 @@
import { test } from '@japa/runner'
import { fileKey, parseSubject } from '#lib/subject/index'
import { parseXrefGroups } from '#lib/nntp/xref'
import { inferTotalParts } from '#lib/yenc/index'
test.group('subject / parseSubject', () => {
test('parses file counter, filename and segment counter', ({ assert }) => {
assert.deepEqual(parseSubject('. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc (56/274)'), {
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
segment: 56,
totalSegments: 274,
fileIndex: 9,
fileTotal: 51,
base: '. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc',
})
})
test('handles a file size after the segment counter', ({ assert }) => {
const subject = '[8/9] - "x.tar.zst.vol063+064.par2" yEnc (37/72) 51315352'
const parsed = parseSubject(subject)!
assert.equal(parsed.segment, 37)
assert.equal(parsed.totalSegments, 72)
assert.equal(parsed.base, '[8/9] - "x.tar.zst.vol063+064.par2" yEnc 51315352')
})
test('ignores counters inside the filename', ({ assert }) => {
const parsed = parseSubject('"Show (1/2) Special.mkv" yEnc (3/10)')!
assert.equal(parsed.name, 'Show (1/2) Special.mkv')
assert.equal(parsed.segment, 3)
assert.equal(parsed.totalSegments, 10)
})
test('the base is equal for all segments of one file', ({ assert }) => {
const a = parseSubject('[1/9] - "a.tar.zst" yEnc (253/1000) 716554110')!
const b = parseSubject('[1/9] - "a.tar.zst" yEnc (254/1000) 716554110')!
assert.equal(a.base, b.base)
})
test('returns null for obfuscated or invalid subjects', ({ assert }) => {
assert.isNull(parseSubject('GZFDJc0hB3Y0FwwAznkk'))
assert.isNull(parseSubject('"name.rar" yEnc'))
assert.isNull(parseSubject('"name.rar" yEnc (0/5)'))
assert.isNull(parseSubject('"name.rar" yEnc (6/5)'))
assert.isNull(parseSubject('"" yEnc (1/5)'))
})
})
test.group('subject / fileKey', () => {
test('differs per poster and per segment count, is stable otherwise', ({ assert }) => {
const key = fileKey('poster@a', '"x.rar" yEnc', 10)
assert.lengthOf(key, 40)
assert.equal(key, fileKey('poster@a', '"x.rar" yEnc', 10))
assert.notEqual(key, fileKey('poster@b', '"x.rar" yEnc', 10))
assert.notEqual(key, fileKey('poster@a', '"x.rar" yEnc', 11))
})
})
test.group('nntp / parseXrefGroups', () => {
test('extracts crossposted groups', ({ assert }) => {
assert.deepEqual(
parseXrefGroups('news.eweka.nl alt.binaries.nl:35801424104 alt.binaries.test:27275430913'),
['alt.binaries.nl', 'alt.binaries.test']
)
assert.deepEqual(parseXrefGroups(undefined), [])
})
})
test.group('yenc / inferTotalParts', () => {
const size = 104857600
const partSize = 384000
test('uses total= when present', ({ assert }) => {
assert.equal(inferTotalParts({ begin: { part: 2, total: 7, line: 128, size, name: 'a' } }), 7)
})
test('derives the total from the part size for middle and last parts', ({ assert }) => {
const expected = Math.ceil(size / partSize) // 274, as in the subject "(56/274)"
assert.equal(expected, 274)
assert.equal(
inferTotalParts({
begin: { part: 56, line: 128, size, name: 'a' },
part: { begin: 21120001, end: 21504000 },
}),
expected
)
assert.equal(
inferTotalParts({
begin: { part: 274, line: 128, size, name: 'a' },
part: { begin: 273 * partSize + 1, end: size },
}),
274
)
})
test('single-part posts have one part', ({ assert }) => {
assert.equal(inferTotalParts({ begin: { line: 128, size: 10, name: 'a' } }), 1)
})
})
test.group('indexer / skipBodyMessageIds', () => {
test('matches the multi-group uploader, not other posts', async ({ assert }) => {
const { default: indexerConfig } = await import('#config/indexer')
const skip = (id: string) => indexerConfig.skipBodyMessageIds.some((re) => re.test(id))
assert.isTrue(skip('<b800526$4db823e$6544066@d810c77fc0.d15fa>'))
assert.isFalse(skip('<Part56of274.4B8CB0C9360F4548AB800FEDCB39CE07@1790532933.local>'))
assert.isFalse(skip('<a6a525b170f44d68a756f1ba6fbd4c60@ngPost>'))
})
})
tests/unit/xover_client.spec.ts
-106
View File
@@ -1,106 +0,0 @@
import { test } from '@japa/runner'
import net from 'node:net'
import { deflateSync } from 'node:zlib'
import { XoverClient, parseOverviews } from '#lib/nntp/xover_client'
import { unzipSync } from 'node:zlib'
const LINES = [
'101\tShow.S01E01 - "a.mkv" yEnc (1/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<one@test>\t\t700000\t5000\tXref: news.example a.b.test:101',
'102\tShow.S01E01 - "a.mkv" yEnc (2/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:01 +0000\t<two@test>\t\t650000\t4600\tXref: news.example a.b.test:102 a.b.other:7',
]
/** A tiny NNTP server: plain replies, gzip after XFEATURE, 420 for 1-1, replies split into small chunks. */
function server(): Promise<{ port: number; close: () => void }> {
return new Promise((resolve) => {
const srv = net.createServer((sock) => {
let gzip = false
const send = (data: Buffer) => {
for (let i = 0; i < data.length; i += 7) sock.write(data.subarray(i, i + 7))
}
sock.write('200 welcome\r\n')
let pending = ''
sock.on('data', (chunk) => {
pending += chunk.toString()
let at: number
while ((at = pending.indexOf('\r\n')) !== -1) {
const line = pending.slice(0, at)
pending = pending.slice(at + 2)
if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
else if (line === 'XFEATURE COMPRESS GZIP') (gzip = true), sock.write('290 Feature Enabled\r\n')
else if (line.startsWith('GROUP')) sock.write('211 2 101 102 a.b.test\r\n')
else if (line === 'XOVER 1-1') sock.write('420 No Articles Selected\r\n')
else if (line.startsWith('XOVER')) {
const body = Buffer.from(LINES.join('\r\n') + '\r\n')
if (gzip) {
sock.write('224 Overview Information Follows [COMPRESS=GZIP]\r\n')
send(Buffer.concat([deflateSync(body), Buffer.from('.\r\n')]))
} else {
sock.write('224 Overview Information Follows\r\n')
send(Buffer.concat([body, Buffer.from('.\r\n')]))
}
} else if (line === 'QUIT') sock.end('205 bye\r\n')
}
})
})
srv.listen(0, '127.0.0.1', () => resolve({ port: (srv.address() as net.AddressInfo).port, close: () => srv.close() }))
})
}
test.group('nntp / XoverClient', () => {
test('parses overview lines into the fields nntp-js gives', ({ assert }) => {
const [[number, fields]] = parseOverviews(LINES[0] + '\r\n')
assert.equal(number, 101)
assert.equal(fields['message-id'], '<one@test>')
assert.equal(fields[':bytes'], '700000')
assert.equal(fields.xref, 'news.example a.b.test:101')
})
for (const compress of [false, true]) {
test(`fetches overviews ${compress ? 'gzip-compressed' : 'plain'}, and an empty range`, async ({ assert }) => {
const { port, close } = await server()
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
try {
assert.equal(client.compressed, compress)
assert.deepEqual(await client.group('a.b.test'), { count: 2, low: 101, high: 102 })
const overviews = await client.xover(101, 102)
assert.deepEqual(overviews.map(([n]) => n), [101, 102])
assert.equal(overviews[1][1].subject, 'Show.S01E01 - "a.mkv" yEnc (2/2)')
assert.deepEqual(await client.xover(1, 1), [])
assert.lengthOf(await client.xover(101, 102), 2)
} finally {
client.close()
close()
}
})
}
test('idle time between commands does not count against the reply timeout', async ({ assert }) => {
const { port, close } = await server()
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress: true, timeoutMs: 200 })
try {
await client.group('a.b.test')
await new Promise((resolve) => setTimeout(resolve, 500))
assert.lengthOf(await client.xover(101, 102), 2)
} finally {
client.close()
close()
}
})
for (const compress of [false, true]) {
test(`hands out overviews as one zlib stream (${compress ? 'as the server sent it' : 'deflated here'})`, async ({ assert }) => {
const { port, close } = await server()
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
try {
await client.group('a.b.test')
const raw = await client.xoverRaw(101, 102)
assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [101, 102])
assert.lengthOf(parseOverviews(unzipSync(await client.xoverRaw(1, 1)).toString('utf8')), 0)
} finally {
client.close()
close()
}
})
}
})
tests/unit/yenc.spec.ts
-192
View File
@@ -1,192 +0,0 @@
import { test } from '@japa/runner'
import yencode from 'yencode'
import {
YencCrcError,
YencError,
YencFile,
crc32Hex,
decodeSegment,
parseYencMeta,
} from '#lib/yenc/index'
/** Builds an article body as a yEnc poster would send it. */
function article(
data: Buffer,
opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string }
) {
const fileSize = opts.fileSize ?? data.length
const encoded = yencode.encode(data).toString('latin1')
const lines: string[] = []
if (opts.part) {
lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`)
const begin = opts.begin ?? 1
lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`)
lines.push(encoded)
const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : ''
lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`)
} else {
lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`)
lines.push(encoded)
lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`)
}
return Buffer.from(lines.join('\r\n'), 'latin1')
}
function randomBytes(length: number, seed = 1) {
const buf = Buffer.alloc(length)
let x = seed
for (let i = 0; i < length; i++) {
x = (x * 1103515245 + 12345) & 0x7fffffff
buf[i] = x & 0xff
}
return buf
}
test.group('yenc / parseYencMeta', () => {
test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => {
const body = Buffer.from(
'=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' +
'=ypart begin=21120001 end=21504000\r\n' +
'xyz\r\n' +
'=yend size=384000 part=56 pcrc32=B6339AFC',
'latin1'
)
const meta = parseYencMeta(body)
assert.deepEqual(meta.begin, {
part: 56,
total: undefined,
line: 128,
size: 104857600,
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
})
assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 })
assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined })
assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz')
})
test('keeps spaces and "=" in the filename', ({ assert }) => {
const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' })
assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo')
})
test('ignores leading blank lines before =ybegin', ({ assert }) => {
const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })])
assert.equal(parseYencMeta(body).begin.name, 'a.txt')
})
test('rejects bodies without yEnc markers', ({ assert }) => {
assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/)
assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/)
})
})
test.group('yenc / decodeSegment', () => {
test('round-trips a single-part post and checks crc32', ({ assert }) => {
const data = randomBytes(5000)
const segment = decodeSegment(article(data, { name: 'single.bin' }))
assert.isTrue(segment.data.equals(data))
assert.equal(segment.offset, 0)
})
test('throws YencCrcError on a corrupted part', ({ assert }) => {
const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 })
const index = body.indexOf('\r\n=ypart') + 60
body[index] = body[index] === 0x41 ? 0x42 : 0x41
assert.throws(() => decodeSegment(body), YencCrcError as any)
})
test('undoes NNTP dot-stuffing', ({ assert }) => {
// Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..".
const data = Buffer.from([4, 5, 6])
const body = Buffer.from(
`=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`,
'latin1'
)
assert.isTrue(decodeSegment(body).data.equals(data))
assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any)
})
})
test.group('yenc / YencFile', () => {
test('assembles parts added out of order and verifies the file crc', ({ assert }) => {
const file = randomBytes(10_000, 7)
const chunk = 3000
const total = Math.ceil(file.length / chunk)
const bodies = Array.from({ length: total }, (_, i) =>
article(file.subarray(i * chunk, (i + 1) * chunk), {
name: 'multi.rar',
fileSize: file.length,
part: i + 1,
total,
begin: i * chunk + 1,
fileCrc: i === total - 1 ? crc32Hex(file) : undefined,
})
)
const segments = bodies.map((body) => decodeSegment(body)).reverse()
const yencFile = YencFile.fromSegment(segments[0])
assert.equal(yencFile.totalParts, 4)
for (const segment of segments.slice(0, -1)) yencFile.add(segment)
assert.isFalse(yencFile.isComplete)
assert.deepEqual(yencFile.missingParts(), [1])
assert.throws(() => yencFile.getBuffer(), /missing parts: 1/)
yencFile.add(segments.at(-1)!)
yencFile.add(segments[0]) // duplicate is ignored
assert.isTrue(yencFile.isComplete)
assert.isTrue(yencFile.getBuffer().equals(file))
})
test('rejects segments of a file with another size', ({ assert }) => {
const a = decodeSegment(article(Buffer.from('aa'), { name: 'a.bin' }))
const b = decodeSegment(article(Buffer.from('bbb'), { name: 'b.bin' }))
assert.throws(() => YencFile.fromSegment(a).add(b), /does not belong to "a.bin"/)
})
test('accepts segments with per-article random names', ({ assert }) => {
const file = randomBytes(4000, 3)
const [x, y] = [0, 1].map((i) =>
decodeSegment(
article(file.subarray(i * 2000, (i + 1) * 2000), {
name: i === 0 ? 'Xq8r2Kd' : 'p0Lm3Zw',
fileSize: 4000,
part: i + 1,
total: 2,
begin: i * 2000 + 1,
})
)
)
const assembled = YencFile.fromSegment(x)
assembled.add(x)
assembled.add(y)
assert.isTrue(assembled.getBuffer().equals(file))
})
})
test.group('content / firstChunkMd5', () => {
test('hashes the first 16 KiB of the decoded first part', async ({ assert }) => {
const { firstChunkMd5 } = await import('#services/ContentService')
const { createHash } = await import('node:crypto')
const file = randomBytes(40_000, 5)
const body = article(file.subarray(0, 20_000), { name: 'x.bin', fileSize: 40_000, part: 1, total: 2 })
assert.equal(firstChunkMd5(body, 40_000), createHash('md5').update(file.subarray(0, 16384)).digest('hex'))
// A first part shorter than 16 KiB of a larger file cannot give the fingerprint.
const short = article(file.subarray(0, 1000), { name: 'x.bin', fileSize: 40_000, part: 1, total: 40 })
assert.isUndefined(firstChunkMd5(short, 40_000))
})
})
test.group('yenc / YencFile without total=', () => {
test('infers the part count when =ybegin has no total', ({ assert }) => {
const file = randomBytes(5000, 11)
const strip = (body: Buffer) => Buffer.from(body.toString('latin1').replace(/ total=\d+/, ''), 'latin1')
const parts = [0, 1].map((i) =>
decodeSegment(strip(article(file.subarray(i * 3000, (i + 1) * 3000), { name: 'n.nzb', fileSize: 5000, part: i + 1, total: 2, begin: i * 3000 + 1 })))
)
const assembled = YencFile.fromSegment(parts[0])
assert.equal(assembled.totalParts, 2)
parts.forEach((p) => assembled.add(p))
assert.isTrue(assembled.getBuffer().equals(file))
})
})