Rebuild: fresh AdonisJS 7 API app on Postgres
The rebuild branch starts over (REBUILD.md): the API starter kit, with Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a shared one in development) and the #lib alias. Phase 1's code is imported from the current app module by module. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
699395c51f
commit
96f0ade10e
264 files changed
+777
-24132
No files matched your search
@@ -1,56 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { FileAggregator, type AggregatableSegment } from '#lib/aggregate/index'
|
||||
|
||||
const seg = (fileKey: string, number: number, totalParts: number, messageId = `${fileKey}-${number}@test`): AggregatableSegment => ({
|
||||
fileKey,
|
||||
number,
|
||||
totalParts,
|
||||
messageId,
|
||||
bytes: 100,
|
||||
})
|
||||
|
||||
test.group('aggregate / FileAggregator', () => {
|
||||
test('completes a file once every part arrived, across batches and out of order', ({ assert }) => {
|
||||
const aggregator = new FileAggregator()
|
||||
assert.deepEqual(aggregator.add([seg('a', 2, 3), seg('b', 1, 1)]).map((f) => f.fileKey), ['b'])
|
||||
assert.equal(aggregator.openFiles, 1)
|
||||
const [done] = aggregator.add([seg('a', 3, 3), seg('a', 1, 3)])
|
||||
assert.equal(done.fileKey, 'a')
|
||||
assert.deepEqual(done.segments.map((s) => s.number), [1, 2, 3])
|
||||
assert.equal(aggregator.openFiles, 0)
|
||||
})
|
||||
|
||||
test('keeps the first article of a part number (a repost) and drops impossible numbers', ({ assert }) => {
|
||||
const aggregator = new FileAggregator()
|
||||
const [done] = aggregator.add([seg('a', 1, 2, 'first@test'), seg('a', 1, 2, 'repost@test'), seg('a', 5, 2), seg('a', 2, 2)])
|
||||
assert.deepEqual(done.segments.map((s) => s.messageId), ['first@test', 'a-2@test'])
|
||||
assert.deepEqual(aggregator.evict().map((s) => s.number), [5])
|
||||
})
|
||||
|
||||
test('gives up idle files as fragments, all of them at the end', ({ assert }) => {
|
||||
const aggregator = new FileAggregator()
|
||||
aggregator.add([seg('old', 1, 3)])
|
||||
aggregator.add([seg('new', 1, 3)])
|
||||
aggregator.add([seg('new', 2, 3)])
|
||||
assert.deepEqual(aggregator.evict(2).map((s) => s.fileKey), ['old'])
|
||||
assert.deepEqual(aggregator.evict(0).map((s) => s.number), [1, 2])
|
||||
assert.equal(aggregator.openFiles, 0)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('seed / runRange', () => {
|
||||
test('a range already past its end only reports that it is done', async ({ assert }) => {
|
||||
const { runRange } = await import('#lib/seed/runner')
|
||||
const batches: any[] = []
|
||||
await runRange({
|
||||
client: { group: async () => assert.fail('no GROUP'), xover: async () => assert.fail('no XOVER') },
|
||||
group: 'a.b.test',
|
||||
start: 101,
|
||||
end: 100,
|
||||
sink: async (batch) => {
|
||||
batches.push(batch)
|
||||
},
|
||||
})
|
||||
assert.deepEqual(batches.map((b) => [b.next, b.headers, b.files.length]), [[101, 0, 0]])
|
||||
})
|
||||
})
|
||||
@@ -1,65 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import yencode from 'yencode'
|
||||
import { isLaterVolume, sniffContentType } from '#lib/content/index'
|
||||
import { crc32Hex, decodeSegment, YencError } from '#lib/yenc/index'
|
||||
|
||||
test.group('content / sniffContentType', () => {
|
||||
test('recognises containers and media by magic bytes', ({ assert }) => {
|
||||
const magic = (hex: string, pad = 16) => Buffer.concat([Buffer.from(hex, 'hex'), Buffer.alloc(pad)])
|
||||
assert.equal(sniffContentType(Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(40)])), 'nzbe')
|
||||
assert.equal(sniffContentType(magic('5041523200504b54')), 'par2')
|
||||
assert.equal(sniffContentType(magic('526172211a070100')), 'rar5')
|
||||
assert.equal(sniffContentType(magic('526172211a0700')), 'rar4')
|
||||
assert.equal(sniffContentType(magic('1a45dfa3')), 'matroska')
|
||||
assert.equal(sniffContentType(magic('0000001866747970')), 'mp4')
|
||||
assert.equal(sniffContentType(magic('28b52ffd')), 'zstd')
|
||||
assert.equal(sniffContentType(Buffer.from('<?xml version="1.0"?>\n<!DOCTYPE nzb PUBLIC "x">')), 'nzb')
|
||||
assert.isNull(sniffContentType(Buffer.from('plain text')))
|
||||
assert.equal(sniffContentType(magic('ffd8ffe0')), 'jpeg')
|
||||
assert.equal(sniffContentType(magic('89504e470d0a1a0a')), 'png')
|
||||
assert.equal(sniffContentType(Buffer.concat([Buffer.from('ID3'), Buffer.alloc(16)])), 'mp3')
|
||||
assert.equal(sniffContentType(Buffer.concat([Buffer.from('fLaC'), Buffer.alloc(16)])), 'flac')
|
||||
assert.equal(sniffContentType(Buffer.from('RIFF\x00\x00\x00\x00AVI LIST', 'latin1')), 'avi')
|
||||
|
||||
// UDF (Blu-ray) volume recognition sequence at 32 KiB
|
||||
const udf = Buffer.alloc(0x9000)
|
||||
udf.write('BEA01', 0x8001, 'latin1')
|
||||
udf.write('NSR03', 0x8801, 'latin1')
|
||||
assert.equal(sniffContentType(udf), 'udf')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / lenientSize', () => {
|
||||
test("recognises yEncBin's encrypted .bin container by its header", ({ assert }) => {
|
||||
// The first 48 bytes of 2141406641.bin (yEncBin@Poster.com).
|
||||
const data = Buffer.from('54f399d117c492696e4dc4ef5b60c11d7f83e92dfdf23450e79370b048123da81e4c8013091b682fb22478daed2ae981', 'hex')
|
||||
assert.equal(sniffContentType(data), 'yencbin')
|
||||
// The second variant (1060238685.bin).
|
||||
assert.equal(sniffContentType(Buffer.from('54f399e917c492696a4e9fb05b60c31d7f83e92df9f16f5086e3c3d9480d66ef1e48d915', 'hex')), 'yencbin')
|
||||
assert.isNull(sniffContentType(Buffer.from('54f399d117c492696e4dc4ef5b60c11e7f83e92d', 'hex')))
|
||||
})
|
||||
|
||||
test('decodes a part whose declared size leaves out a header, still checking the CRC', ({ assert }) => {
|
||||
const data = Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(16, 7), Buffer.from('ciphertext-here')])
|
||||
const declared = data.length - 20
|
||||
const body = Buffer.from(
|
||||
`=ybegin part=1 line=128 size=${declared} name=f6dd6b156692a52b.bin\r\n=ypart begin=1 end=${declared}\r\n` +
|
||||
yencode.encode(data).toString('latin1') +
|
||||
`\r\n=yend size=${declared} part=1 pcrc32=${crc32Hex(data)}`,
|
||||
'latin1'
|
||||
)
|
||||
assert.throws(() => decodeSegment(body), YencError as any)
|
||||
assert.isTrue(decodeSegment(body, { lenientSize: true }).data.equals(data))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('content / isLaterVolume', () => {
|
||||
test('recognises later RAR volumes and par2 recovery volumes, also under a random stem', ({ assert }) => {
|
||||
for (const name of ['P8vttYaPec1mL4nDZHPZo7Nd6iEWj8lP.part47.rar', 'Show.S01E01.part002.rar', 'abc.r00', 'abc.r123', 'x7Kq9.vol03+04.par2', 'Show.vol127-128.par2']) {
|
||||
assert.isTrue(isLaterVolume(name), name)
|
||||
}
|
||||
for (const name of ['P8vttYaPec1mL4nDZHPZo7Nd6iEWj8lP.part01.rar', 'abc.part1.rar', 'abc.rar', 'x7Kq9.par2', '6e13186238ab54ca498e46d2d77da099', 'abc.7z.002', 'abc.mkv']) {
|
||||
assert.isFalse(isLaterVolume(name), name)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,14 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
|
||||
test.group('nntp / articleAtDate', () => {
|
||||
test('finds the article posted at a time, skipping gaps', async ({ assert }) => {
|
||||
// One article a second from t0, articles 1,000,000–1,100,000; 1,040,000–1,040,999 expired.
|
||||
const t0 = Date.UTC(2026, 8, 1)
|
||||
const dateOf = async (n: bigint) => (n >= 1_040_000n && n < 1_041_000n ? null : t0 + Number(n - 1_000_000n) * 1000)
|
||||
const at = await articleAtDate(1_000_000n, 1_100_000n, t0 + 60_000 * 1000, dateOf, 100n)
|
||||
assert.isAtLeast(Number(at), 1_059_900)
|
||||
assert.isAtMost(Number(at), 1_060_000)
|
||||
assert.equal(await articleAtDate(1_000_000n, 1_100_000n, t0 - 1, dateOf, 100n), 1_000_000n)
|
||||
})
|
||||
})
|
||||
@@ -1,52 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { delimitedReader } from '#lib/nntp/delimited_reader'
|
||||
|
||||
const CRLF = Buffer.from('\r\n')
|
||||
const END = Buffer.from('\r\n.\r\n')
|
||||
|
||||
async function* chunks(...parts: string[]) {
|
||||
for (const part of parts) yield Buffer.from(part, 'latin1')
|
||||
}
|
||||
|
||||
test.group('nntp / delimitedReader', () => {
|
||||
test('reads an empty multi-line reply (XOVER of a range without articles)', async ({ assert }) => {
|
||||
const reader = delimitedReader(chunks('224 Overview follows\r\n.\r\n', '211 1 2 3 g\r\n'), CRLF)
|
||||
assert.equal((await reader.next()).value!.toString(), '224 Overview follows')
|
||||
assert.equal((await reader.next(END)).value!.toString(), '')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '211 1 2 3 g')
|
||||
})
|
||||
|
||||
test('reads multi-line data, also with the terminator split over chunks', async ({ assert }) => {
|
||||
for (const split of [1, 2, 3, 4, 5, 6, 7]) {
|
||||
const reply = '224 ok\r\nline 1\r\n..dotted\r\nline 3\r\n.\r\n200 next\r\n'
|
||||
const at = reply.indexOf('.\r\n200') - 2 + split - 1
|
||||
const reader = delimitedReader(chunks(reply.slice(0, at), reply.slice(at)), CRLF)
|
||||
assert.equal((await reader.next()).value!.toString(), '224 ok')
|
||||
assert.equal((await reader.next(END)).value!.toString(), 'line 1\r\n..dotted\r\nline 3', `split ${split}`)
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '200 next')
|
||||
}
|
||||
})
|
||||
|
||||
test('reads replies byte by byte', async ({ assert }) => {
|
||||
const text = '200 welcome\r\n224 ok\r\na\r\n.\r\n224 ok\r\n.\r\n211 x\r\n'
|
||||
const reader = delimitedReader(chunks(...text.split('')), CRLF)
|
||||
assert.equal((await reader.next()).value!.toString(), '200 welcome')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
|
||||
assert.equal((await reader.next(END)).value!.toString(), 'a')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '224 ok')
|
||||
assert.equal((await reader.next(END)).value!.toString(), '')
|
||||
assert.equal((await reader.next(CRLF)).value!.toString(), '211 x')
|
||||
})
|
||||
|
||||
test('keeps binary data intact', async ({ assert }) => {
|
||||
const body = Buffer.from([0, 13, 255, 10, 46, 13, 46, 10, 13])
|
||||
const source = (async function* () {
|
||||
yield Buffer.from('222 body\r\n')
|
||||
yield body
|
||||
yield Buffer.from('\r\n.\r\n')
|
||||
})()
|
||||
const reader = delimitedReader(source, CRLF)
|
||||
await reader.next()
|
||||
assert.deepEqual((await reader.next(END)).value, body)
|
||||
})
|
||||
})
|
||||
@@ -1,46 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { zstdCompressSync, constants } from 'node:zlib'
|
||||
import { NICOVIDEO_FILE, nicovideoInfo, nicovideoReleaseName, zstdPrefix } from '#lib/content/index'
|
||||
|
||||
function tarEntry(name: string, data: Buffer): Buffer {
|
||||
const header = Buffer.alloc(512)
|
||||
header.write(name, 0, 'utf8')
|
||||
header.write(data.length.toString(8).padStart(11, '0') + '\0', 124, 'latin1')
|
||||
header.write('ustar\0', 257, 'latin1')
|
||||
return Buffer.concat([header, data, Buffer.alloc((512 - (data.length % 512)) % 512)])
|
||||
}
|
||||
|
||||
test.group('nicovideo captures', () => {
|
||||
test('reads the title from the start of a capture, even a truncated one', async ({ assert }) => {
|
||||
const dir = '1790720489_nicovideo_jp_watch_sm27965722'
|
||||
const response = {
|
||||
data: {
|
||||
metadata: { jsonLds: [{ author: { name: 'まふまふ' } }] },
|
||||
response: {
|
||||
video: { id: 'sm27965722', title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】', registeredAt: '2016-01-08T20:00:00+09:00' },
|
||||
owner: { nickname: 'まふまふ' },
|
||||
tag: { items: [{ name: '音楽' }, { name: 'After_the_Rain' }] },
|
||||
},
|
||||
},
|
||||
}
|
||||
const tar = Buffer.concat([
|
||||
tarEntry(`${dir}/`, Buffer.alloc(0)),
|
||||
tarEntry(`${dir}/server_response.metadata`, Buffer.from(JSON.stringify(response, null, 2))),
|
||||
tarEntry(`${dir}/01.cmfa`, randomBytes(300_000)),
|
||||
])
|
||||
const compressed = zstdCompressSync(tar, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
|
||||
// Only the first part of the stream, as in the file's first article.
|
||||
const info = nicovideoInfo(await zstdPrefix(compressed.subarray(0, Math.floor(compressed.length * 0.6))))!
|
||||
assert.deepEqual(info, {
|
||||
id: 'sm27965722',
|
||||
title: '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】',
|
||||
owner: 'まふまふ',
|
||||
registeredAt: '2016-01-08T20:00:00+09:00',
|
||||
tags: ['音楽', 'After_the_Rain'],
|
||||
})
|
||||
assert.equal(nicovideoReleaseName(info, 'sm1'), '桜花ニ月夜ト袖シグレ／After the Rain【そらる×まふまふ】 - まふまふ (nicovideo sm27965722)')
|
||||
assert.equal(NICOVIDEO_FILE.exec(`${dir}.tar.zst`)![1], 'sm27965722')
|
||||
assert.isNull(nicovideoInfo(await zstdPrefix(Buffer.from('not zstd'))))
|
||||
})
|
||||
})
|
||||
@@ -1,18 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { parseGroupResponse } from '#lib/nntp/group'
|
||||
|
||||
test.group('nntp / parseGroupResponse', () => {
|
||||
test('parses the reply from news.eweka.nl', ({ assert }) => {
|
||||
// Captured 2026-09-27. count == last - first + 1, so the field order is standard.
|
||||
assert.deepEqual(parseGroupResponse('211 27275164317 266599 27275430915 alt.binaries.test'), {
|
||||
count: 27275164317n,
|
||||
first: 266599n,
|
||||
last: 27275430915n,
|
||||
name: 'alt.binaries.test',
|
||||
})
|
||||
})
|
||||
|
||||
test('rejects non-211 replies', ({ assert }) => {
|
||||
assert.throws(() => parseGroupResponse('411 No such group'), /Unexpected GROUP response/)
|
||||
})
|
||||
})
|
||||
@@ -1,67 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
|
||||
test.group('nzb / buildNzb', () => {
|
||||
test('writes files, groups and ordered segments', ({ assert }) => {
|
||||
const xml = buildNzb(
|
||||
[
|
||||
{
|
||||
name: 'Some.Show.S01E01.mkv',
|
||||
poster: 'poster@example.invalid',
|
||||
date: 1_790_000_000_000,
|
||||
groups: ['alt.binaries.test'],
|
||||
segments: [{ number: 2, id: 'b@x', size: 700 }, { number: 1, id: 'a@x', size: 800 }],
|
||||
},
|
||||
],
|
||||
{ title: 'Some.Show.S01E01' }
|
||||
)
|
||||
assert.include(xml, '<!DOCTYPE nzb PUBLIC "-//newzBin//DTD NZB 1.1//EN"')
|
||||
assert.include(xml, '<meta type="name">Some.Show.S01E01</meta>')
|
||||
assert.include(xml, 'subject=""Some.Show.S01E01.mkv" yEnc (1/2)"')
|
||||
assert.include(xml, 'date="1790000000"')
|
||||
assert.include(xml, '<group>alt.binaries.test</group>')
|
||||
assert.isBelow(xml.indexOf('>a@x<'), xml.indexOf('>b@x<'))
|
||||
assert.include(xml, '<segment bytes="800" number="1">a@x</segment>')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('nzb / parseNzb', () => {
|
||||
test('reads meta, files, groups and segments, decoding entities', async ({ assert }) => {
|
||||
const { parseNzb } = await import('#lib/nzb/index')
|
||||
const xml = `<?xml version="1.0" encoding="utf-8" ?>
|
||||
<!DOCTYPE nzb PUBLIC "-//newzBin//DTD NZB 0.9//EN" "http://www.newzbin.com/DTD/nzb/nzb-0.9.dtd">
|
||||
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
|
||||
<head><meta type="name">Some.Show.S01E01</meta><meta type="password">s3cr&t</meta></head>
|
||||
<file poster="Yenc@power-post.org (Cosmo71)" date="1790598923" subject="Wond40 "Nachbe" x.part1.rar" yEnc (0001/2)">
|
||||
<groups><group>alt.binaries.bloaf</group></groups>
|
||||
<segments>
|
||||
<segment bytes="1000" number="2">b@x.local</segment>
|
||||
<segment bytes="2000" number="1"><a@x.local></segment>
|
||||
</segments>
|
||||
</file>
|
||||
<file poster="p" date="1" subject="empty"><groups/><segments></segments></file>
|
||||
</nzb>`
|
||||
const nzb = parseNzb(xml)!
|
||||
assert.deepEqual(nzb.meta, { name: 'Some.Show.S01E01', password: 's3cr&t' })
|
||||
assert.lengthOf(nzb.files, 1)
|
||||
assert.equal(nzb.files[0].subject, 'Wond40 "Nachbe" x.part1.rar" yEnc (0001/2)')
|
||||
assert.deepEqual(nzb.files[0].groups, ['alt.binaries.bloaf'])
|
||||
assert.deepEqual(nzb.files[0].segments, [
|
||||
{ number: 1, bytes: 2000, id: 'a@x.local' },
|
||||
{ number: 2, bytes: 1000, id: 'b@x.local' },
|
||||
])
|
||||
assert.isNull(parseNzb('<html>not an nzb</html>'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('nzb / stored segment lists', () => {
|
||||
test('pack and unpack round-trip, in order, also with gaps in the numbering', async ({ assert }) => {
|
||||
const { packSegments, unpackSegments } = await import('#lib/nzb/segments')
|
||||
const sequential = Array.from({ length: 300 }, (_, i) => ({ number: i + 1, id: `Ab${i}Cd-1790600369${i}@nyuu`, size: i === 299 ? 1234 : 740000 }))
|
||||
assert.deepEqual(unpackSegments(packSegments([...sequential].reverse())), sequential)
|
||||
const gaps = [{ number: 3, id: 'c@x', size: 3 }, { number: 1, id: 'a@x', size: 1 }, { number: 7, id: 'g@x', size: 7 }]
|
||||
assert.deepEqual(unpackSegments(packSegments(gaps)), [gaps[1], gaps[0], gaps[2]])
|
||||
assert.deepEqual(unpackSegments(packSegments([])), [])
|
||||
assert.isBelow(packSegments(sequential).length / sequential.length, 30)
|
||||
})
|
||||
})
|
||||
@@ -1,60 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
|
||||
const md5 = (data: Buffer) => createHash('md5').update(data).digest()
|
||||
|
||||
/** Builds one PAR2 packet with a valid checksum. */
|
||||
function packet(setId: Buffer, type: string, body: Buffer) {
|
||||
const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
|
||||
const length = Buffer.alloc(8)
|
||||
length.writeBigUInt64LE(BigInt(64 + padded.length))
|
||||
const typeBuf = Buffer.alloc(16)
|
||||
typeBuf.write(type, 'latin1')
|
||||
const hashed = Buffer.concat([setId, typeBuf, padded])
|
||||
return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
|
||||
}
|
||||
|
||||
function fileDesc(setId: Buffer, name: string, size: number) {
|
||||
const len = Buffer.alloc(8)
|
||||
len.writeBigUInt64LE(BigInt(size))
|
||||
const md5_16k = md5(Buffer.from(`${name}-16k`))
|
||||
const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
|
||||
const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
|
||||
return packet(setId, 'PAR 2.0\0FileDesc', body)
|
||||
}
|
||||
|
||||
test.group('par2 / parsePar2', () => {
|
||||
const setId = Buffer.alloc(16, 7)
|
||||
|
||||
test('reads file names and sizes from FileDesc packets', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)),
|
||||
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600),
|
||||
fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800),
|
||||
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common
|
||||
])
|
||||
assert.isTrue(isPar2(data))
|
||||
const set = parsePar2(data)!
|
||||
assert.equal(set.recoverySetId, setId.toString('hex'))
|
||||
assert.deepEqual(
|
||||
set.files.map((f) => [f.name, f.length]),
|
||||
[
|
||||
['Some.Movie.2024.part01.rar', 104857600],
|
||||
['Some.Movie.2024.part02.rar', 52428800],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('skips corrupted packets and resumes at the next one', ({ assert }) => {
|
||||
const bad = fileDesc(setId, 'bad.bin', 1)
|
||||
bad[100] ^= 0xff
|
||||
const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)])
|
||||
assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin'])
|
||||
})
|
||||
|
||||
test('returns null for non-par2 data', ({ assert }) => {
|
||||
assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1')))
|
||||
assert.isNull(parsePar2(Buffer.from('hello world')))
|
||||
})
|
||||
})
|
||||
@@ -1,86 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { crc32Hex } from '#lib/yenc/index'
|
||||
import { probeInspection, type InspectTask } from '#lib/content/probe'
|
||||
|
||||
/** One yEnc article of `file`: part `index` of `size` bytes (escaping '.' and whitespace too, so no dot-stuffing). */
|
||||
function yencArticle(name: string, file: Buffer, index: number, size: number) {
|
||||
const data = file.subarray(index * size, (index + 1) * size)
|
||||
const total = Math.ceil(file.length / size)
|
||||
const out: number[] = []
|
||||
for (const byte of data) {
|
||||
const c = (byte + 42) & 0xff
|
||||
if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
|
||||
else out.push(c)
|
||||
}
|
||||
const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
|
||||
const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
|
||||
return Buffer.from(head + Buffer.from(out).toString('latin1') + tail, 'latin1')
|
||||
}
|
||||
|
||||
/** A posted file as articles, and its inspection task. */
|
||||
function posted(name: string, file: Buffer, size = 400) {
|
||||
const articles = new Map<string, Buffer>()
|
||||
const segments = []
|
||||
for (let i = 0; i * size < file.length; i++) {
|
||||
const id = `${name}-${i}@test`
|
||||
articles.set(id, yencArticle(name, file, i, size))
|
||||
segments.push({ number: i + 1, id, size: size + 100 })
|
||||
}
|
||||
const fetched: string[] = []
|
||||
const fetch = async (id: string) => {
|
||||
fetched.push(id)
|
||||
const body = articles.get(id)
|
||||
if (!body) throw new Error(`430 no such article ${id}`)
|
||||
return body
|
||||
}
|
||||
const task: InspectTask = { kind: 'inspect', fileId: 1, name, segments, sevenZip: { volumes: 1, last: segments } }
|
||||
return { task, fetch, fetched }
|
||||
}
|
||||
|
||||
// `7z a -pSecret -mhe=on`: an encrypted header (tests/unit/sevenzip.spec.ts).
|
||||
const HEADER_ENCRYPTED = Buffer.from(
|
||||
'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
|
||||
'base64'
|
||||
)
|
||||
|
||||
test.group('content probe', () => {
|
||||
test('a first segment gives size, fingerprint and the yEnc name', async ({ assert }) => {
|
||||
const file = Buffer.alloc(20_000, 7)
|
||||
const { task, fetch, fetched } = posted('Some.Name.bin', file, 17_000)
|
||||
const probe = await probeInspection(task, fetch)
|
||||
assert.include(probe, { size: 20_000, contentType: null, yencName: 'Some.Name.bin' })
|
||||
assert.match(probe.md5_16k!, /^[0-9a-f]{32}$/)
|
||||
assert.deepEqual(fetched, ['Some.Name.bin-0@test'])
|
||||
})
|
||||
|
||||
test('an NZB is read whole', async ({ assert }) => {
|
||||
const xml = `<?xml version="1.0"?>\n<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">${'<file subject="x"/>'.repeat(60)}</nzb>`
|
||||
const { task, fetch, fetched } = posted('x.nzb', Buffer.from(xml))
|
||||
const probe = await probeInspection(task, fetch)
|
||||
assert.equal(probe.contentType, 'nzb')
|
||||
assert.equal(probe.text, xml)
|
||||
// Every segment once: the first, decoded to sniff, is reused for the whole.
|
||||
assert.deepEqual(fetched, task.segments.map((s) => s.id))
|
||||
})
|
||||
|
||||
test('a 7z archive with encrypted headers is known from its tail, without bsdtar', async ({ assert }) => {
|
||||
const { task, fetch } = posted('a.7z', HEADER_ENCRYPTED, 100)
|
||||
const probe = await probeInspection(task, fetch)
|
||||
assert.equal(probe.contentType, '7z')
|
||||
assert.include(probe.archive!, { format: '7z', headersEncrypted: true })
|
||||
assert.lengthOf(probe.archive!.entries, 0)
|
||||
})
|
||||
|
||||
test('a missing first article is a result, not a failure (no retries)', async ({ assert }) => {
|
||||
const { task, fetch } = posted('gone.bin', Buffer.alloc(10, 1))
|
||||
task.segments[0].id = 'missing@test'
|
||||
assert.include(await probeInspection(task, fetch), { contentType: 'missing' })
|
||||
// Through the seed relay: 410.
|
||||
const relayed = await probeInspection(task, async () => {
|
||||
throw new Error('Seed API /api/v1/seed/articles?id=x: 410 {"error":"No such article"}')
|
||||
})
|
||||
assert.equal(relayed.contentType, 'missing')
|
||||
// Anything else still fails (and is retried).
|
||||
await assert.rejects(() => probeInspection(task, async () => { throw new Error('NNTP BODY: no answer within 120 s') }), /no answer/)
|
||||
})
|
||||
})
|
||||
@@ -1,376 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { crc32 } from 'node:zlib'
|
||||
import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, walkRar, type RarEntry, type VolumeReader } from '#lib/rar/index'
|
||||
import { crc32Hex } from '#lib/yenc/index'
|
||||
|
||||
/** Entries without the extraction fields, for comparing listings. */
|
||||
const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest)
|
||||
|
||||
// --- RAR 4 builders -----------------------------------------------------------
|
||||
|
||||
const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
|
||||
|
||||
function rar4Block(type: number, flags: number, body: Buffer) {
|
||||
const header = Buffer.alloc(7)
|
||||
header.writeUInt8(type, 2)
|
||||
header.writeUInt16LE(flags, 3)
|
||||
header.writeUInt16LE(7 + body.length, 5)
|
||||
const block = Buffer.concat([header, body])
|
||||
block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0)
|
||||
return block
|
||||
}
|
||||
|
||||
function rar4Main(flags: number) {
|
||||
return rar4Block(0x73, flags, Buffer.alloc(6))
|
||||
}
|
||||
|
||||
function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) {
|
||||
const body = Buffer.alloc(25)
|
||||
body.writeUInt8(opts.method ?? 0x33, 18)
|
||||
body.writeUInt32LE(opts.packed, 0)
|
||||
body.writeUInt32LE(opts.unpacked, 4)
|
||||
body.writeUInt16LE(name.length, 19)
|
||||
// LONG_BLOCK (0x8000) is always set on file headers.
|
||||
return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)])
|
||||
}
|
||||
|
||||
// --- RAR 5 builders -----------------------------------------------------------
|
||||
|
||||
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
||||
|
||||
function vint(value: number) {
|
||||
const bytes: number[] = []
|
||||
do {
|
||||
let byte = value % 128
|
||||
value = Math.floor(value / 128)
|
||||
if (value > 0) byte |= 0x80
|
||||
bytes.push(byte)
|
||||
} while (value > 0)
|
||||
return Buffer.from(bytes)
|
||||
}
|
||||
|
||||
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
|
||||
let flags = headerFlags
|
||||
const parts = [vint(type)]
|
||||
const sizes: Buffer[] = []
|
||||
if (opts.extra) {
|
||||
flags |= 0x0001
|
||||
sizes.push(vint(opts.extra.length))
|
||||
}
|
||||
if (opts.data !== undefined) {
|
||||
flags |= 0x0002
|
||||
sizes.push(vint(opts.data))
|
||||
}
|
||||
const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
|
||||
const sized = Buffer.concat([vint(header.length), header])
|
||||
const crc = Buffer.alloc(4)
|
||||
crc.writeUInt32LE(crc32(sized) >>> 0)
|
||||
return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
|
||||
}
|
||||
|
||||
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
|
||||
const nameBuf = Buffer.from(name, 'utf8')
|
||||
const fields = Buffer.concat([
|
||||
vint(opts.dir ? 0x0001 : 0), // file flags
|
||||
vint(size),
|
||||
vint(0x20), // attributes
|
||||
vint(0), // compression info
|
||||
vint(0), // host OS
|
||||
vint(nameBuf.length),
|
||||
nameBuf,
|
||||
])
|
||||
// Encryption record: size, type 1, then its data (dummy here).
|
||||
const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
|
||||
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
|
||||
}
|
||||
|
||||
test.group('rar / RAR 4', () => {
|
||||
test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
RAR4,
|
||||
rar4Main(0x0001 | 0x0100), // volume, first volume
|
||||
rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }),
|
||||
])
|
||||
assert.equal(rarFormat(data), 'rar4')
|
||||
const info = parseRarHeaders(data)!
|
||||
assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true })
|
||||
assert.deepEqual(listing(info.entries), [
|
||||
{ name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
|
||||
])
|
||||
})
|
||||
|
||||
test('detects password-protected files and encrypted headers', ({ assert }) => {
|
||||
const files = parseRarHeaders(
|
||||
Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })])
|
||||
)!
|
||||
assert.isTrue(files.entries[0].encrypted)
|
||||
const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))!
|
||||
assert.isTrue(headers.headersEncrypted)
|
||||
assert.lengthOf(headers.entries, 0)
|
||||
})
|
||||
|
||||
test('copies out stored entries, not compressed ones', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
RAR4,
|
||||
rar4Main(0),
|
||||
rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }),
|
||||
rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }),
|
||||
])
|
||||
const [stored, packed] = parseRarHeaders(data)!.entries
|
||||
data.write('<nzb>', stored.dataOffset)
|
||||
assert.equal(storedEntryData(data, stored)?.toString(), '<nzb>')
|
||||
assert.isFalse(packed.stored)
|
||||
assert.isNull(storedEntryData(data, packed))
|
||||
})
|
||||
|
||||
test('stops at a header with a bad CRC', ({ assert }) => {
|
||||
const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 })
|
||||
const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 })
|
||||
bad[10] ^= 0xff
|
||||
const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))!
|
||||
assert.deepEqual(info.entries.map((e) => e.name), ['a.bin'])
|
||||
})
|
||||
|
||||
test('decodes compressed Unicode names', ({ assert }) => {
|
||||
// "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte,
|
||||
// then 3 plain bytes), followed by one more flag byte for the last byte.
|
||||
const ascii = Buffer.from('?.txt', 'latin1')
|
||||
const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74])
|
||||
assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt')
|
||||
assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rar / RAR 5', () => {
|
||||
test('lists files and reads volume flags', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
RAR5,
|
||||
rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume
|
||||
rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }),
|
||||
])
|
||||
assert.equal(rarFormat(data), 'rar5')
|
||||
const info = parseRarHeaders(data)!
|
||||
assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false })
|
||||
assert.deepEqual(listing(info.entries), [
|
||||
{ name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true },
|
||||
])
|
||||
})
|
||||
|
||||
test('a later volume carries a volume number', ({ assert }) => {
|
||||
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))!
|
||||
assert.isFalse(info.firstVolume)
|
||||
})
|
||||
|
||||
test('detects encrypted files and encrypted headers', ({ assert }) => {
|
||||
const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))!
|
||||
assert.isTrue(files.entries[0].encrypted)
|
||||
const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))!
|
||||
assert.isTrue(headers.headersEncrypted)
|
||||
})
|
||||
|
||||
test('marks the listing complete at the end-of-archive header', ({ assert }) => {
|
||||
const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))!
|
||||
assert.isFalse(info.truncated)
|
||||
assert.lengthOf(info.entries, 1)
|
||||
})
|
||||
|
||||
test('copies out entries stored without compression', ({ assert }) => {
|
||||
const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))])
|
||||
const entry = parseRarHeaders(data)!.entries[0]
|
||||
assert.isTrue(entry.stored)
|
||||
data.write('<nzb>', entry.dataOffset)
|
||||
assert.equal(storedEntryData(data, entry)?.toString(), '<nzb>')
|
||||
assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry))
|
||||
assert.isNull(storedEntryData(data, { ...entry, encrypted: true }))
|
||||
})
|
||||
|
||||
test('returns null for other data', ({ assert }) => {
|
||||
assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1')))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rar / release naming from contents', () => {
|
||||
const info = (names: string[]) =>
|
||||
({ format: 'rar5', volume: false, firstVolume: true, solid: false, headersEncrypted: false, truncated: true,
|
||||
entries: names.map((name) => ({ name, size: 1, packedSize: 1, directory: false, encrypted: false, splitBefore: false, splitAfter: false })) }) as any
|
||||
|
||||
test('uses the shared top-level directory when it looks like a release name', async ({ assert }) => {
|
||||
const { releaseNameFromContents } = await import('#services/ArchiveService')
|
||||
assert.equal(
|
||||
releaseNameFromContents(info(['GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/00-a.jpg', 'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/01-b.flac'])),
|
||||
'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM'
|
||||
)
|
||||
// No directory: the largest readable entry names the release.
|
||||
assert.equal(
|
||||
releaseNameFromContents(info(['Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265.nzb'])),
|
||||
'Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265'
|
||||
)
|
||||
assert.isUndefined(releaseNameFromContents(info(['movie.mkv'])))
|
||||
assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91PqA7.mkv'])))
|
||||
assert.isUndefined(releaseNameFromContents(info(['a/x', 'b/y'])))
|
||||
assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91Pq/x.mkv'])))
|
||||
})
|
||||
|
||||
test('hidden entries (a decoy) name nothing', async ({ assert }) => {
|
||||
const { releaseNameFromContents, isHidden } = await import('#services/ArchiveService')
|
||||
const decoy = '.1917-2019-2160p_MA_WEB-DL_Hybrid_H265_DV_HDR_DDP_Atmos_5.1_English-HONE-.mkv'
|
||||
const listing = info([decoy, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX', `${decoy}/${decoy}.19OMnL`, 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX/boston.blue.s01e02.rar'])
|
||||
listing.entries[0].directory = true
|
||||
listing.entries[1].directory = true
|
||||
listing.entries[2].size = 22_093_429_730
|
||||
assert.equal(releaseNameFromContents(listing), 'Boston_Blue_S01E02_1080p_AMZN_WEB-DL_DDP5_1_H_264-FLUX')
|
||||
assert.isTrue(isHidden(`${decoy}/x.mkv`))
|
||||
assert.isTrue(isHidden('dir/.hidden.mkv'))
|
||||
assert.isFalse(isHidden('Some.Release/movie.mkv'))
|
||||
})
|
||||
|
||||
test('recognises obfuscated names', async ({ assert }) => {
|
||||
const { looksRandom } = await import('#services/ArchiveService')
|
||||
assert.isTrue(looksRandom('CGBWgdhlSCAOAW62gLWRRM'))
|
||||
assert.isTrue(looksRandom('0220e975b1417f96b77dc64410e10ebc'))
|
||||
assert.isFalse(looksRandom('Plus.and.Minus.S01E06.1080p'))
|
||||
assert.isFalse(looksRandom('1790546808_nicovideo_jp_watch_sm26679715'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('rar / looksRandom capitals', () => {
|
||||
test('long letter-only tokens with many capitals are random', async ({ assert }) => {
|
||||
const { looksRandom } = await import('#services/ArchiveService')
|
||||
assert.isTrue(looksRandom('RmISakytFwXSCEzjVEpaKk'))
|
||||
assert.isTrue(looksRandom('326678d1d07ef12a.bin'))
|
||||
assert.isFalse(looksRandom('Some.Show.S01E01'))
|
||||
assert.isFalse(looksRandom('TheQuickBrownFoxJumps'))
|
||||
})
|
||||
})
|
||||
|
||||
// --- Archive walks (D40) --------------------------------------------------------
|
||||
|
||||
/** Volumes split into segments of `segmentSize` bytes; counts the segments read. */
|
||||
function memoryReader(volumes: Buffer[], segmentSize: number) {
|
||||
const read = new Set<string>()
|
||||
const reader: VolumeReader & { segments: Set<string> } = {
|
||||
segments: read,
|
||||
volumes: volumes.length,
|
||||
size: (v) => volumes[v].length,
|
||||
async read(v, offset, more) {
|
||||
const index = Math.floor(offset / segmentSize)
|
||||
if (offset >= volumes[v].length) return null
|
||||
read.add(`${v}:${index}`)
|
||||
if (more) read.add(`${v}:${index + 1}`)
|
||||
return volumes[v].subarray(offset, (index + (more ? 2 : 1)) * segmentSize)
|
||||
},
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
function rar5Set() {
|
||||
const main = (n: number) => rar5Block(1, 0, n === 0 ? vint(0x0001) : Buffer.concat([vint(0x0003), vint(n)]))
|
||||
const end = rar5Block(5, 0, vint(0x0001))
|
||||
const SPLIT_BEFORE = 0x0008
|
||||
const SPLIT_AFTER = 0x0010
|
||||
return [
|
||||
Buffer.concat([RAR5, main(0), rar5File('Book/01.mp3', 3000, { packed: 3000 }), rar5File('Book/02.mp3', 7000, { packed: 5000, headerFlags: SPLIT_AFTER }), end]),
|
||||
Buffer.concat([
|
||||
RAR5,
|
||||
main(1),
|
||||
rar5File('Book/02.mp3', 7000, { packed: 2000, headerFlags: SPLIT_BEFORE }),
|
||||
rar5File('Book/03.mp3', 1000, { packed: 1000 }),
|
||||
rar5File('Book/04.mp3', 4500, { packed: 4000, headerFlags: SPLIT_AFTER }),
|
||||
end,
|
||||
]),
|
||||
Buffer.concat([RAR5, main(2), rar5File('Book/04.mp3', 4500, { packed: 500, headerFlags: SPLIT_BEFORE }), rar5File('Book/05.mp3', 100, { packed: 100 }), rar5Block(5, 0, vint(0))]),
|
||||
]
|
||||
}
|
||||
|
||||
test.group('rar / archive walks', () => {
|
||||
test('lists every entry of a RAR5 set, reading a segment per header', async ({ assert }) => {
|
||||
// 700-byte segments: headers get cut off by segment ends.
|
||||
const reader = memoryReader(rar5Set(), 700)
|
||||
const walk = await walkRar(reader)
|
||||
assert.isTrue(walk.complete)
|
||||
assert.equal(walk.format, 'rar5')
|
||||
assert.deepEqual(walk.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3', 'Book/03.mp3', 'Book/04.mp3', 'Book/05.mp3'])
|
||||
assert.equal(walk.entries[1].size, 7000)
|
||||
const total = rar5Set().reduce((sum, v) => sum + Math.ceil(v.length / 700), 0)
|
||||
assert.isBelow(reader.segments.size, total / 2)
|
||||
})
|
||||
|
||||
test('lists a RAR4 set across volumes', async ({ assert }) => {
|
||||
const end = rar4Block(0x7b, 0, Buffer.alloc(0))
|
||||
const volumes = [
|
||||
Buffer.concat([RAR4, rar4Main(0x0001 | 0x0100), rar4File(Buffer.from('a.flac'), { packed: 900, unpacked: 900 }), rar4File(Buffer.from('b.flac'), { packed: 2000, unpacked: 2500, flags: 0x0002 }), end]),
|
||||
Buffer.concat([RAR4, rar4Main(0x0001), rar4File(Buffer.from('b.flac'), { packed: 500, unpacked: 2500, flags: 0x0001 }), rar4File(Buffer.from('c.flac'), { packed: 10, unpacked: 10 }), end]),
|
||||
]
|
||||
const walk = await walkRar(memoryReader(volumes, 512))
|
||||
assert.isTrue(walk.complete)
|
||||
assert.deepEqual(walk.entries.map((e) => e.name), ['a.flac', 'b.flac', 'c.flac'])
|
||||
})
|
||||
|
||||
test('stops at a missing segment or a broken header, keeping what it found', async ({ assert }) => {
|
||||
const set = rar5Set()
|
||||
const missing = memoryReader(set, 700)
|
||||
const read = missing.read.bind(missing)
|
||||
missing.read = async (v, offset, more) => (v === 1 ? null : read(v, offset, more))
|
||||
const partial = await walkRar(missing)
|
||||
assert.equal(partial.stopped, 'unreadable')
|
||||
assert.deepEqual(partial.entries.map((e) => e.name), ['Book/01.mp3', 'Book/02.mp3'])
|
||||
|
||||
const broken = rar5Set()
|
||||
// The header of 03.mp3 in volume 2, after 02.mp3's continuation and its data.
|
||||
const at = broken[1].indexOf(Buffer.from('Book/03.mp3')) - 4
|
||||
broken[1][at] ^= 0xff
|
||||
const invalid = await walkRar(memoryReader(broken, 700))
|
||||
assert.equal(invalid.stopped, 'invalid')
|
||||
assert.lengthOf(invalid.entries, 2)
|
||||
})
|
||||
|
||||
test('finds segments from their yEnc offsets', async ({ assert }) => {
|
||||
const { SegmentReader } = await import('#lib/rar/index')
|
||||
const set = rar5Set()
|
||||
const size = 700
|
||||
const articles = new Map<string, Buffer>()
|
||||
const files = set.map((volume, v) => {
|
||||
const segments = []
|
||||
for (let i = 0; i * size < volume.length; i++) {
|
||||
const id = `v${v}s${i}@test`
|
||||
articles.set(id, yencArticle(`x.part${v + 1}.rar`, volume, i, size))
|
||||
segments.push({ number: i + 1, id, size: 0 })
|
||||
}
|
||||
return segments
|
||||
})
|
||||
const reader = new SegmentReader(files, 100, async (id) => articles.get(id)!)
|
||||
const walk = await walkRar(reader)
|
||||
assert.isTrue(walk.complete)
|
||||
assert.lengthOf(walk.entries, 5)
|
||||
assert.isBelow(reader.fetched, articles.size / 2)
|
||||
|
||||
const capped = new SegmentReader(files, 3, async (id) => articles.get(id)!)
|
||||
const stopped = await walkRar(capped)
|
||||
assert.equal(stopped.stopped, 'unreadable')
|
||||
assert.equal(capped.fetched, 3)
|
||||
})
|
||||
})
|
||||
|
||||
/** One yEnc article of `file`: part `index` of `size` bytes. */
|
||||
function yencArticle(name: string, file: Buffer, index: number, size: number) {
|
||||
const data = file.subarray(index * size, (index + 1) * size)
|
||||
const total = Math.ceil(file.length / size)
|
||||
const out: number[] = []
|
||||
for (const byte of data) {
|
||||
const c = (byte + 42) & 0xff
|
||||
// Escape the critical characters, and '.' and whitespace so no line needs dot-stuffing.
|
||||
if ([0x00, 0x0a, 0x0d, 0x3d, 0x2e, 0x09, 0x20].includes(c)) out.push(0x3d, (c + 64) & 0xff)
|
||||
else out.push(c)
|
||||
}
|
||||
const lines: string[] = []
|
||||
const encoded = Buffer.from(out)
|
||||
for (let i = 0; i < encoded.length; i += 128) {
|
||||
let end = Math.min(encoded.length, i + 128)
|
||||
if (encoded[end - 1] === 0x3d && end < encoded.length) end++
|
||||
lines.push(encoded.subarray(i, end).toString('latin1'))
|
||||
i = end - 128
|
||||
}
|
||||
const head = `=ybegin part=${index + 1} total=${total} line=128 size=${file.length} name=${name}\r\n=ypart begin=${index * size + 1} end=${index * size + data.length}\r\n`
|
||||
const tail = `\r\n=yend size=${data.length} part=${index + 1} pcrc32=${crc32Hex(data)}\r\n`
|
||||
return Buffer.from(head + lines.join('\r\n') + tail, 'latin1')
|
||||
}
|
||||
@@ -1,169 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { fileStem, releaseIdentity, subjectRemainder } from '#lib/release/index'
|
||||
import { looksLikeFilename, weakFileKey } from '#lib/subject/index'
|
||||
|
||||
test.group('release / fileStem', () => {
|
||||
test('reduces all files of a set to the same stem', ({ assert }) => {
|
||||
const names = [
|
||||
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst',
|
||||
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.par2',
|
||||
'1790546808_nicovideo_jp_watch_sm26679715.tar.zst.vol03+04.par2',
|
||||
]
|
||||
assert.deepEqual(new Set(names.map(fileStem)), new Set(['1790546808_nicovideo_jp_watch_sm26679715']))
|
||||
})
|
||||
|
||||
test('handles rar, split and par2 volume naming', ({ assert }) => {
|
||||
assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0')
|
||||
assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265')
|
||||
assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264')
|
||||
assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('Some.Movie.2024.nfo'), 'Some.Movie.2024')
|
||||
assert.equal(fileStem('00-artist-album-web-2026.m3u'), '00-artist-album-web-2026')
|
||||
assert.equal(fileStem('f6dd6b156692a52b.bin'), 'f6dd6b156692a52b')
|
||||
})
|
||||
|
||||
test('never reduces a name to nothing', ({ assert }) => {
|
||||
assert.equal(fileStem('.par2'), '.par2')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / subjectRemainder', () => {
|
||||
test('is empty when the subject only has counters, filename and size', ({ assert }) => {
|
||||
assert.equal(subjectRemainder('[1/4] - "x.tar.zst" yEnc 15411409'), '')
|
||||
assert.equal(subjectRemainder('. - [00/88] - "60SHkTQSvn5RZGExwju2.nzb" yEnc'), '')
|
||||
})
|
||||
|
||||
test('keeps a release name written outside the quotes', ({ assert }) => {
|
||||
assert.equal(
|
||||
subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'),
|
||||
'Some Show S01E02 1080p'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / releaseIdentity', () => {
|
||||
test('groups a par2 set with its archive by stem, per poster', ({ assert }) => {
|
||||
const base = { poster: 'p', weak: false }
|
||||
const a = releaseIdentity({ ...base, subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
|
||||
const b = releaseIdentity({ ...base, subject: '[3/4] - "x.tar.zst.vol01+02.par2" yEnc 2', name: 'x.tar.zst.vol01+02.par2' })!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.name, 'x')
|
||||
assert.equal(a.fileTotal, 4)
|
||||
const other = releaseIdentity({ ...base, poster: 'q', subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })!
|
||||
assert.notEqual(a.key, other.key)
|
||||
})
|
||||
|
||||
test('a par2 index posted without [i/M] counter joins its release', ({ assert }) => {
|
||||
const base = { poster: 'nicovideo_archiver', weak: false }
|
||||
const archive = releaseIdentity({ ...base, subject: '[1/6] - "x.tar.zst" yEnc 105054576', name: 'x.tar.zst' })!
|
||||
const index = releaseIdentity({ ...base, subject: '"x.tar.zst.par2" yEnc 3100', name: 'x.tar.zst.par2' })!
|
||||
assert.equal(index.key, archive.key)
|
||||
assert.equal(archive.fileTotal, 6)
|
||||
assert.isUndefined(index.fileTotal)
|
||||
})
|
||||
|
||||
test('uses the subject text when it names the release', ({ assert }) => {
|
||||
const base = { poster: 'p', weak: false }
|
||||
const a = releaseIdentity({ ...base, subject: 'My.Release [01/40] - "abc.part01.rar" yEnc', name: 'abc.part01.rar' })!
|
||||
const b = releaseIdentity({ ...base, subject: 'My.Release [40/40] - "abc.par2" yEnc', name: 'abc.par2' })!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.name, 'My Release')
|
||||
})
|
||||
|
||||
test('obfuscated volumes group by stem regardless of the random poster', ({ assert }) => {
|
||||
const a = releaseIdentity({ poster: '3Uzx7TTSFpLzz@ngPost.com', subject: '"lJzu.part033.rar" yEnc', name: 'lJzu.part033.rar', weak: true })!
|
||||
const b = releaseIdentity({ poster: 'bGLd1oaXRdrK1@ngPost.com', subject: '"lJzu.part034.rar" yEnc', name: 'lJzu.part034.rar', weak: true })!
|
||||
assert.equal(a.key, b.key)
|
||||
})
|
||||
|
||||
test('random extensionless names cannot be grouped by name', ({ assert }) => {
|
||||
const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'
|
||||
assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true }))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('subject / weakFileKey', () => {
|
||||
test('ignores random per-article names, keeps real filenames', ({ assert }) => {
|
||||
assert.isFalse(looksLikeFilename('204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231'))
|
||||
assert.isTrue(looksLikeFilename('lJzuoIKA3T8xjzAD3vqtw0.part033.rar'))
|
||||
// Same file, different random names per article: same key.
|
||||
assert.equal(weakFileKey('aaaa1111', 5000, 7), weakFileKey('bbbb2222', 5000, 7))
|
||||
// Equal-sized volumes with real names: different keys.
|
||||
assert.notEqual(weakFileKey('x.part01.rar', 5000, 7), weakFileKey('x.part02.rar', 5000, 7))
|
||||
assert.notEqual(weakFileKey('aaaa1111', 5000, 7), weakFileKey('aaaa1111', 5001, 7))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / rarReleaseIdentity', () => {
|
||||
test('groups volumes by the shared top-level directory', async ({ assert }) => {
|
||||
const { rarReleaseIdentity } = await import('#lib/release/index')
|
||||
const first = rarReleaseIdentity([
|
||||
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/00-cover.jpg', size: 589382 },
|
||||
{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/01-track.flac', size: 30_000_000 },
|
||||
])!
|
||||
const later = rarReleaseIdentity([{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/03-track.flac', size: 29_000_000 }])!
|
||||
assert.equal(first.key, later.key)
|
||||
assert.equal(first.name, 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM')
|
||||
assert.equal(first.source, 'rar')
|
||||
})
|
||||
|
||||
test('without a directory, volumes of one file share its first entry', async ({ assert }) => {
|
||||
const { rarReleaseIdentity } = await import('#lib/release/index')
|
||||
const a = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
|
||||
const b = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.name, 'Some.Show.S01E01.1080p')
|
||||
assert.isNull(rarReleaseIdentity([]))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / namesRelease', () => {
|
||||
test('random per-file tags and poster ids are not a release name', async ({ assert }) => {
|
||||
const { namesRelease, subjectRemainder, releaseIdentity } = await import('#lib/release/index')
|
||||
const subject = 'elys-de81bcabef5faf6a - [Nyh5Q07Y] "JkpNIFg7En50zU1U" yEnc'
|
||||
assert.isFalse(namesRelease(subjectRemainder(subject)))
|
||||
const a = releaseIdentity({ poster: 'elys', subject, name: 'JkpNIFg7En50zU1U', weak: false })!
|
||||
assert.equal(a.source, 'stem')
|
||||
assert.isTrue(namesRelease(subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc')))
|
||||
assert.isTrue(namesRelease('My Release'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / post id', () => {
|
||||
test('groups random-named files of one post by poster, post id and file count', async ({ assert }) => {
|
||||
const { releaseIdentity, isPostToken } = await import('#lib/release/index')
|
||||
const base = { poster: 'jW344UtC8R2bZEegrVLz@x', weak: false }
|
||||
const a = releaseIdentity({ ...base, subject: '83110629-n [10/18] - "eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k" yEnc', name: 'eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k' })!
|
||||
const b = releaseIdentity({ ...base, subject: '83110629-n [01/18] - "2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0" yEnc', name: '2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0' })!
|
||||
assert.equal(a.key, b.key)
|
||||
assert.equal(a.source, 'post')
|
||||
assert.equal(a.fileTotal, 18)
|
||||
const other = releaseIdentity({ ...base, subject: '83110630-n [01/18] - "x" yEnc', name: 'x' })!
|
||||
assert.notEqual(a.key, other.key)
|
||||
|
||||
assert.isTrue(isPostToken('83110629 n'))
|
||||
assert.isFalse(isPostToken('PRiVATE'))
|
||||
assert.isFalse(isPostToken(''))
|
||||
// Without a counter there is nothing to bound the post: fall back to the stem.
|
||||
assert.equal(releaseIdentity({ ...base, subject: '83110629-n - "x.bin" yEnc', name: 'x.bin' })!.source, 'stem')
|
||||
})
|
||||
})
|
||||
|
||||
test.group('release / par2 names', () => {
|
||||
test('names a season pack after what its episodes share', async ({ assert }) => {
|
||||
const { par2ReleaseName } = await import('#lib/release/index')
|
||||
const eps = Array.from({ length: 20 }, (_, i) =>
|
||||
`SEAL.Team.S03E${String(i + 1).padStart(2, '0')}.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato.mkv`)
|
||||
assert.equal(par2ReleaseName(eps), 'SEAL.Team.S03.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato')
|
||||
// Movie plus sample and nfo: the movie's (longest) name.
|
||||
assert.equal(
|
||||
par2ReleaseName(['Some.Movie.2024.1080p.BluRay.x264-GRP.mkv', 'Some.Movie.2024.1080p.BluRay.x264-GRP.nfo', 'sample.mkv']),
|
||||
'Some.Movie.2024.1080p.BluRay.x264-GRP'
|
||||
)
|
||||
// Numbered parts: the shared name without the part number.
|
||||
assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP')
|
||||
assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024')
|
||||
})
|
||||
})
|
||||
@@ -1,48 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import net from 'node:net'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import nntpConfig from '#config/nntp'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import SeedFetchPool from '#services/SeedFetchPool'
|
||||
import { parseOverviews } from '#lib/nntp/xover_client'
|
||||
|
||||
test.group('SeedFetchPool', () => {
|
||||
test('a chunk that fails (dead connection, stalled compressed reply) is fetched again uncompressed', async ({ assert }) => {
|
||||
let connections = 0
|
||||
const compressAsked: Record<number, boolean> = {}
|
||||
const server = net.createServer((sock) => {
|
||||
const n = ++connections
|
||||
sock.write('200 welcome\r\n')
|
||||
sock.on('data', (chunk) => {
|
||||
for (const line of chunk.toString().split('\r\n').filter(Boolean)) {
|
||||
if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
|
||||
else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
|
||||
else if (line.startsWith('XFEATURE')) (compressAsked[n] = true), sock.write('500 no\r\n')
|
||||
else if (line.startsWith('GROUP')) sock.write('211 1 7 7 a.b.test\r\n')
|
||||
// The first connection dies as the request arrives, like one the server dropped while idle.
|
||||
else if (line.startsWith('XOVER') && n === 1) sock.destroy()
|
||||
else if (line.startsWith('XOVER')) sock.write('224 overviews\r\n7\tsubject\tposter\tdate\t<a@b>\t\t10\t1\r\n.\r\n')
|
||||
}
|
||||
})
|
||||
})
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
const saved = { host: nntpConfig.host, port: nntpConfig.port, secure: nntpConfig.secure, connections: indexerConfig.seed.nntpConnections, compress: indexerConfig.seed.nntpCompress }
|
||||
// The fallback from a compressed pooled connection (the default).
|
||||
indexerConfig.seed.nntpCompress = true
|
||||
Object.assign(nntpConfig, { host: '127.0.0.1', port: (server.address() as net.AddressInfo).port, secure: false })
|
||||
indexerConfig.seed.nntpConnections = 2
|
||||
try {
|
||||
const raw = await SeedFetchPool.fetch('a.b.test', 7, 7)
|
||||
assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [7])
|
||||
assert.equal(connections, 2)
|
||||
// The pooled connection asked for gzip; the retry didn't.
|
||||
assert.deepEqual(compressAsked, { 1: true })
|
||||
} finally {
|
||||
SeedFetchPool.closeIdle()
|
||||
Object.assign(nntpConfig, { host: saved.host, port: saved.port, secure: saved.secure })
|
||||
indexerConfig.seed.nntpConnections = saved.connections
|
||||
indexerConfig.seed.nntpCompress = saved.compress
|
||||
server.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,16 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { cut, uncovered } from '#lib/seed/plan'
|
||||
|
||||
test.group('seed plan', () => {
|
||||
test('plans only what earlier ranges leave uncovered', ({ assert }) => {
|
||||
// a.b.ath on 2026-10-06: test ranges around 49.8B and a plan below them; the backfill ends at 58.4B.
|
||||
assert.deepEqual(uncovered(1, 100, [[40, 50], [10, 20], [45, 60]]), [[1, 9], [21, 39], [61, 100]])
|
||||
assert.deepEqual(uncovered(1, 100, []), [[1, 100]])
|
||||
assert.deepEqual(uncovered(1, 100, [[0, 200]]), [])
|
||||
assert.deepEqual(uncovered(10, 20, [[1, 5], [25, 30]]), [[10, 20]])
|
||||
})
|
||||
|
||||
test('cuts gaps into ranges, newest first', ({ assert }) => {
|
||||
assert.deepEqual(cut([[1, 9], [61, 100]], 25), [[76, 100], [61, 75], [1, 9]])
|
||||
})
|
||||
})
|
||||
@@ -1,95 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import {
|
||||
checkHeader,
|
||||
encodedHeaderEncrypted,
|
||||
listingStart,
|
||||
parseBsdtarListing,
|
||||
parseSevenZipStart,
|
||||
readNumber,
|
||||
sevenZipVolume,
|
||||
} from '#lib/sevenzip/index'
|
||||
|
||||
// `7z a tiny.7z Tiny.Release` (Tiny.Release/a.txt): an LZMA-encoded header.
|
||||
const ENCODED = Buffer.from(
|
||||
'N3q8ryccAATeIVLCfQAAAAAAAAAhAAAAAAAAAGkLW/MBAAtoZWxsbyB3b3JsZAoAAACBMweuD87ysgwHyEN/QbH6/duIttdfpOXRhZbi6t7tycGOUDyW1l2aTE7xmuZ/NAXKgg599VxKYAF1XImsfT9LzHQKaEH6PEZambLnwJM62mXXEJIpaYvhCrHFeQv+jLnYjgJMfwZEcAAAABcGEAEJbQAHCwEAASMDAQEFXQAQAAAMgJYKASTou9wAAA==',
|
||||
'base64'
|
||||
)
|
||||
// The same with -mhc=off: a plain header.
|
||||
const PLAIN = Buffer.from(
|
||||
'N3q8ryccAASpRwRsEAAAAAAAAACWAAAAAAAAACTou9wBAAtoZWxsbyB3b3JsZAoAAQQGAAEJEAAHCwEAASEhAQAMDAAICgEtOwivAAAFAg4BgBkJAAAAAAAAAAAAEUEAVABpAG4AeQAuAFIAZQBsAGUAYQBzAGUAAABUAGkAbgB5AC4AUgBlAGwAZQBhAHMAZQAvAGEALgB0AHgAdAAAABkCAAAUEgEA/QYYLBBR3QH9BhgsEFHdARUKAQAQgP1BIIC0gQAA',
|
||||
'base64'
|
||||
)
|
||||
|
||||
// `7z a -pSecret -mhe=on`: an encrypted header; with -mhe=off only the file data is encrypted.
|
||||
const HEADER_ENCRYPTED = Buffer.from(
|
||||
'N3q8ryccAARtHJcDoAAAAAAAAAA+AAAAAAAAAGqzNxflXzLifhRjmRlgXvMEO9zhcADrwDAjtLT4COwZQUsqMkFT0jGipxcTP5MNbocZW7iGLP5PCAuqfouDxAoI2Zw2AqMRF2m/XOVYRokWM4bHuH73xp1LIva5JpyAQSViuI66Jf5xxemaGnSvoC5/mHDHe6VZUxQwMmH3QuTR15+loSuNhbEU2feP1E2EdwIvHSYXdJ9jDqHe6XjW3gfh/2+NFwYQAQmAkAAHCwEAAiQG8QcBElMPIu8lRTeoM0YvSGcM+B6l/SMDAQEFXQAQAAABAAyAgoCmCgGDcQMzAAA=',
|
||||
'base64'
|
||||
)
|
||||
const DATA_ENCRYPTED = Buffer.from(
|
||||
'N3q8ryccAASBXGg3kQAAAAAAAAAiAAAAAAAAAO4xlWU16N8rTbvoF1dbq3eLPdZiAACBMweuD87ysgwHsMPa919FiphISvi++OI2giqcUPCAdmXzZtkqFSqP0Nr4rWaRH5dR+vNA2jJGIbDAZpbQM/ka9oh27tkqKb/+dHyx1SFHkxbLyWvd7W0qwaDHxSvW2fTpRdJSmi74crF957Qwy+P8dDF0MOYr5AGHAQHJSAAAFwYQAQmAgQAHCwEAASMDAQEFXQAQAAAMgKYKAaUZQIYAAA==',
|
||||
'base64'
|
||||
)
|
||||
const headerOf = (archive: Buffer) => {
|
||||
const start = parseSevenZipStart(archive)!
|
||||
return archive.subarray(start.headerStart, start.archiveSize)
|
||||
}
|
||||
|
||||
test.group('sevenzip', () => {
|
||||
test('tells encrypted headers from the encoded header alone', ({ assert }) => {
|
||||
assert.isTrue(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED)))
|
||||
assert.isFalse(encodedHeaderEncrypted(headerOf(DATA_ENCRYPTED)))
|
||||
assert.isFalse(encodedHeaderEncrypted(headerOf(ENCODED)))
|
||||
// A plain header isn't encoded: bsdtar lists it.
|
||||
assert.isNull(encodedHeaderEncrypted(headerOf(PLAIN)))
|
||||
assert.isNull(encodedHeaderEncrypted(headerOf(HEADER_ENCRYPTED).subarray(0, 12)))
|
||||
})
|
||||
|
||||
test('finds the header at the end from the signature header', ({ assert }) => {
|
||||
const start = parseSevenZipStart(ENCODED)!
|
||||
assert.deepEqual(start, { headerStart: 157, headerSize: 33, headerCrc: start.headerCrc, archiveSize: 190 })
|
||||
const header = ENCODED.subarray(start.headerStart, start.archiveSize)
|
||||
assert.isTrue(checkHeader(start, header))
|
||||
assert.isFalse(checkHeader(start, Buffer.alloc(33)))
|
||||
// Encoded: the packed header right after the file data (packPos 16) is needed too.
|
||||
assert.equal(listingStart(start, header), 48)
|
||||
|
||||
const plain = parseSevenZipStart(PLAIN)!
|
||||
assert.equal(plain.archiveSize, PLAIN.length)
|
||||
assert.equal(listingStart(plain, PLAIN.subarray(plain.headerStart)), plain.headerStart)
|
||||
})
|
||||
|
||||
test('rejects other data and a damaged signature header', ({ assert }) => {
|
||||
assert.isNull(parseSevenZipStart(Buffer.from('Rar!\x1a\x07\x01\x00'.padEnd(40, '\0'), 'latin1')))
|
||||
const damaged = Buffer.from(ENCODED)
|
||||
damaged[14] ^= 1
|
||||
assert.isNull(parseSevenZipStart(damaged))
|
||||
})
|
||||
|
||||
test('reads 7z numbers', ({ assert }) => {
|
||||
assert.deepEqual(readNumber(Buffer.from([0x10]), 0), [16, 1])
|
||||
assert.deepEqual(readNumber(Buffer.from([0x81, 0x02]), 0), [0x102, 2])
|
||||
assert.deepEqual(readNumber(Buffer.from([0xc0, 0x34, 0x12]), 0), [0x1234, 3])
|
||||
assert.deepEqual(readNumber(Buffer.from([0xff, 1, 0, 0, 0, 0, 0, 0, 0]), 0), [1, 9])
|
||||
})
|
||||
|
||||
test('parses bsdtar listings', ({ assert }) => {
|
||||
const out = [
|
||||
'drwxrwxr-x 0 0 0 0 Sep 30 19:13 Some.Release.2026/',
|
||||
'-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release.2026/movie.mkv',
|
||||
'-rw-rw-r-- 0 0 0 3 Jan 1 2019 Some.Release.2026/some info.nfo',
|
||||
'',
|
||||
].join('\n')
|
||||
assert.deepEqual(parseBsdtarListing(out), [
|
||||
{ name: 'Some.Release.2026', size: 0, directory: true },
|
||||
{ name: 'Some.Release.2026/movie.mkv', size: 3000000, directory: false },
|
||||
{ name: 'Some.Release.2026/some info.nfo', size: 3, directory: false },
|
||||
])
|
||||
})
|
||||
|
||||
test('numbers volumes', ({ assert }) => {
|
||||
assert.equal(sevenZipVolume('x.7z.001'), 1)
|
||||
assert.equal(sevenZipVolume('X.7Z.012'), 12)
|
||||
assert.equal(sevenZipVolume('x.7z'), 0)
|
||||
assert.isNull(sevenZipVolume('x.part01.rar'))
|
||||
})
|
||||
})
|
||||
@@ -1,162 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { splitInstances, splitVerified, verifyVolume, type SplitArticle } from '#lib/split/index'
|
||||
import { crc32Hex } from '#lib/yenc/index'
|
||||
|
||||
/** Deterministic pseudo-random numbers. */
|
||||
function rng(seed: number) {
|
||||
let x = seed
|
||||
return () => {
|
||||
x = (x * 1103515245 + 12345) & 0x7fffffff
|
||||
return x / 0x7fffffff
|
||||
}
|
||||
}
|
||||
|
||||
interface PostOptions {
|
||||
volumes: number
|
||||
parts: number
|
||||
/** Parallel upload connections; they finish slightly out of order. */
|
||||
connections?: number
|
||||
/** First article number of the post. */
|
||||
start?: number
|
||||
/** Unrelated articles between two articles of this post (other posters). */
|
||||
noise?: number
|
||||
/** Fraction of articles that never arrive. */
|
||||
drop?: number
|
||||
label?: string
|
||||
seed?: number
|
||||
}
|
||||
|
||||
/** Simulates a posting tool uploading equal-sized volumes. */
|
||||
function post(opts: PostOptions): SplitArticle[] {
|
||||
const random = rng(opts.seed ?? 1)
|
||||
const connections = opts.connections ?? 1
|
||||
const label = opts.label ?? 'p'
|
||||
const queue: { volume: number; number: number }[] = []
|
||||
for (let v = 0; v < opts.volumes; v++) for (let n = 1; n <= opts.parts; n++) queue.push({ volume: v, number: n })
|
||||
|
||||
const articles: SplitArticle[] = []
|
||||
let articleNumber = opts.start ?? 1000
|
||||
for (let i = 0; i < queue.length; i += connections) {
|
||||
// One round: each connection posts one article; they land in random order.
|
||||
const round = queue.slice(i, i + connections).sort(() => random() - 0.5)
|
||||
for (const { volume, number } of round) {
|
||||
articleNumber += 1 + Math.floor(random() * (opts.noise ?? 0))
|
||||
if (random() < (opts.drop ?? 0)) continue
|
||||
articles.push({ number, messageId: `${label}${volume}.${number}`, bytes: 700_000, articleNumber })
|
||||
}
|
||||
}
|
||||
return articles
|
||||
}
|
||||
|
||||
/** Source volume of an article, from its simulated message-id. */
|
||||
const source = (a: SplitArticle) => a.messageId.split('.')[0]
|
||||
|
||||
/** Every result volume must contain articles of one source volume only. */
|
||||
function assertPure(assert: any, volumes: SplitArticle[][]) {
|
||||
for (const volume of volumes) {
|
||||
assert.lengthOf(new Set(volume.map(source)), 1, `mixed volume: ${volume.map((a) => a.messageId).join(' ')}`)
|
||||
}
|
||||
}
|
||||
|
||||
test.group('split / splitInstances', () => {
|
||||
test('separates sequentially posted volumes', ({ assert }) => {
|
||||
const volumes = splitInstances(post({ volumes: 3, parts: 14 }), { totalParts: 14 })
|
||||
assert.lengthOf(volumes, 3)
|
||||
assertPure(assert, volumes)
|
||||
assert.isTrue(volumes.every((v) => v.length === 14))
|
||||
})
|
||||
|
||||
test('separates volumes posted over interleaved connections, out of order', ({ assert }) => {
|
||||
for (const seed of [1, 2, 3, 4, 5]) {
|
||||
const volumes = splitInstances(post({ volumes: 6, parts: 40, connections: 8, noise: 30, seed }), { totalParts: 40 })
|
||||
assertPure(assert, volumes)
|
||||
assert.lengthOf(volumes, 6, `seed ${seed}`)
|
||||
}
|
||||
})
|
||||
|
||||
test('a volume that completes is never mixed, even with heavy loss', ({ assert }) => {
|
||||
for (const seed of [11, 12, 13, 14, 15, 16, 17, 18]) {
|
||||
const articles = post({ volumes: 5, parts: 30, connections: 6, noise: 20, drop: 0.2, seed })
|
||||
const volumes = splitInstances(articles, { totalParts: 30 })
|
||||
assertPure(assert, volumes.filter((v) => v.length === 30))
|
||||
}
|
||||
})
|
||||
|
||||
test('a missing article only affects its own volume', ({ assert }) => {
|
||||
const articles = post({ volumes: 4, parts: 20, connections: 4, drop: 0.05, seed: 7 })
|
||||
const volumes = splitInstances(articles, { totalParts: 20 })
|
||||
assertPure(assert, volumes)
|
||||
assert.lengthOf(volumes, 4)
|
||||
})
|
||||
|
||||
test('posts of the same size at different times do not mix', ({ assert }) => {
|
||||
// An earlier post that lost articles (e.g. before the backfill window), then two more.
|
||||
const articles = [
|
||||
...post({ volumes: 3, parts: 14, connections: 4, drop: 0.3, label: 'a', start: 1_000, seed: 3 }),
|
||||
...post({ volumes: 3, parts: 14, connections: 4, label: 'b', start: 200_000, seed: 4 }),
|
||||
...post({ volumes: 2, parts: 14, connections: 2, label: 'c', start: 900_000, seed: 5 }),
|
||||
]
|
||||
const volumes = splitInstances(articles, { totalParts: 14 })
|
||||
// The damaged post may produce mixed volumes, but those stay incomplete
|
||||
// (they never become files); every volume that completes is pure.
|
||||
const complete = volumes.filter((v) => v.length === 14)
|
||||
assertPure(assert, complete)
|
||||
assert.includeMembers(complete.map((v) => source(v[0])), ['b0', 'b1', 'b2', 'c0', 'c1'])
|
||||
// The later, intact posts are not affected at all.
|
||||
assertPure(assert, volumes.filter((v) => !source(v[0]).startsWith('a')))
|
||||
})
|
||||
|
||||
test('ignores duplicate message-ids and does not depend on input order', ({ assert }) => {
|
||||
const articles = post({ volumes: 2, parts: 5, connections: 3, seed: 9 })
|
||||
const shuffled = [...articles].reverse().concat(articles[0])
|
||||
assert.deepEqual(splitInstances(shuffled, { totalParts: 5 }), splitInstances(articles, { totalParts: 5 }))
|
||||
})
|
||||
})
|
||||
|
||||
/** Like post(), with real data per volume so part and file CRCs are genuine. */
|
||||
function postWithCrcs(opts: PostOptions): SplitArticle[] {
|
||||
const random = rng((opts.seed ?? 1) + 1000)
|
||||
const size = 64
|
||||
const data = new Map<string, Buffer>()
|
||||
const articles = post(opts)
|
||||
for (const a of articles) {
|
||||
const volume = source(a)
|
||||
if (!data.has(volume)) {
|
||||
data.set(volume, Buffer.from(Array.from({ length: opts.parts * size }, () => Math.floor(random() * 256))))
|
||||
}
|
||||
const file = data.get(volume)!
|
||||
a.pcrc32 = crc32Hex(file.subarray((a.number - 1) * size, a.number * size))
|
||||
a.partSize = size
|
||||
a.fileCrc32 = a.number === opts.parts ? crc32Hex(file) : null
|
||||
}
|
||||
return articles
|
||||
}
|
||||
|
||||
test.group('split / verified', () => {
|
||||
test('verifyVolume accepts a correct volume and rejects a mixed one', ({ assert }) => {
|
||||
const articles = postWithCrcs({ volumes: 2, parts: 6 })
|
||||
const volumeA = articles.filter((a) => source(a) === 'p0')
|
||||
assert.isTrue(verifyVolume(volumeA, 6))
|
||||
const mixed = volumeA.map((a) => (a.number === 3 ? articles.find((b) => source(b) === 'p1' && b.number === 3)! : a))
|
||||
assert.isFalse(verifyVolume(mixed, 6))
|
||||
assert.isFalse(verifyVolume(volumeA.slice(0, 5), 6))
|
||||
})
|
||||
|
||||
test('only verified volumes come out, and they are never mixed', ({ assert }) => {
|
||||
for (const seed of [21, 22, 23, 24, 25, 26]) {
|
||||
const articles = [
|
||||
...postWithCrcs({ volumes: 4, parts: 30, connections: 16, noise: 20, drop: 0.05, label: 'a', seed }),
|
||||
...postWithCrcs({ volumes: 3, parts: 30, connections: 16, noise: 20, label: 'b', start: 300_000, seed: seed + 50 }),
|
||||
]
|
||||
for (const volume of splitVerified(articles, { totalParts: 30 })) {
|
||||
assert.isTrue(verifyVolume(volume, 30))
|
||||
assertPure(assert, [volume])
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('recovers every volume of a cleanly posted set', ({ assert }) => {
|
||||
const volumes = splitVerified(postWithCrcs({ volumes: 5, parts: 20, connections: 4, noise: 10, seed: 31 }), { totalParts: 20 })
|
||||
assert.lengthOf(volumes, 5)
|
||||
})
|
||||
})
|
||||
@@ -1,186 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { deflateRawSync } from 'node:zlib'
|
||||
import {
|
||||
decodeSpotBinary,
|
||||
decodeSpotText,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
spotCategoryLabels,
|
||||
verifySpotHeader,
|
||||
} from '#lib/spotnet/index'
|
||||
import { ubbToText } from '#lib/newznab/xml'
|
||||
|
||||
// A real spot from free.pt (self-signed, key 7), 2026-09-28.
|
||||
const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "<jWCbaAa4f1A7LC6agmOEx@spot.net>", "date": "Mon, 28 Sep 2026 18:24:45 GMT"}
|
||||
|
||||
test.group('spotnet / headers', () => {
|
||||
test('parses a spot header', ({ assert }) => {
|
||||
const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)!
|
||||
assert.include(spot, {
|
||||
poster: 'Bart',
|
||||
title: 'The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi',
|
||||
tag: 'Bart',
|
||||
category: 0,
|
||||
keyId: 7,
|
||||
size: 21651233204,
|
||||
messageId: 'jWCbaAa4f1A7LC6agmOEx@spot.net',
|
||||
})
|
||||
assert.deepEqual(spot.subcats, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] })
|
||||
})
|
||||
|
||||
test('verifies its signatures, and rejects another message id', ({ assert }) => {
|
||||
const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)!
|
||||
assert.isTrue(verifySpotHeader(spot))
|
||||
assert.isFalse(verifySpotHeader({ ...spot, messageId: 'other@spot.net' }))
|
||||
assert.isNull(parseSpotHeader('someone <x@y>', 'subject', '<a@b>'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('spotnet / xml and nzb', () => {
|
||||
test('reads the NZB segments and category from the X-XML lines', ({ assert }) => {
|
||||
const raw = [
|
||||
'From: Bart <x@y>',
|
||||
'X-XML: <Spotnet><Posting><Key>7</Key><Title>A & B</Title><Category>01<Sub>01a09</Sub>',
|
||||
'X-XML: <Sub>01z00</Sub></Category><NZB><Segment>abc@spot.net</Segment><Segment>def@spot.net</Segment></NZB>',
|
||||
'X-XML: <Size>123</Size></Posting></Spotnet>',
|
||||
].join('\r\n')
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw)
|
||||
assert.equal(headers.from, 'Bart <x@y>')
|
||||
const spot = parseSpotXml(xml)!
|
||||
assert.deepEqual(spot.nzbSegments, ['abc@spot.net', 'def@spot.net'])
|
||||
assert.include(spot, { title: 'A & B', category: 0, size: 123 })
|
||||
assert.deepEqual(spot.subcats, ['01a09', '01z00'])
|
||||
})
|
||||
|
||||
test('reads the description, tag and image', ({ assert }) => {
|
||||
const spot = parseSpotXml(
|
||||
`<Spotnet><Posting><Key>7</Key><Tag>Bart</Tag><Title>T</Title>` +
|
||||
`<Description>Lily Tomlin[br][br]If you're ‘in’ [url=https://example.com/a]here[/url]</Description>` +
|
||||
`<Image Width='185' Height='278'><Segment>FYe6l94kQsY7LC6agh5Xn@spot.net</Segment></Image>` +
|
||||
`<Category>01<Sub>01a09</Sub></Category></Posting></Spotnet>`
|
||||
)!
|
||||
assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]')
|
||||
assert.equal(spot.tag, 'Bart')
|
||||
assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 })
|
||||
// Older spots link an image on the web.
|
||||
assert.deepEqual(parseSpotXml('<Posting><Image>http://x.nl/a.jpg</Image></Posting>')!.image, { url: 'http://x.nl/a.jpg' })
|
||||
assert.isNull(parseSpotXml('<Posting><Title>T</Title></Posting>')!.image)
|
||||
})
|
||||
|
||||
test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => {
|
||||
const raw = [
|
||||
'From: vanker',
|
||||
' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>',
|
||||
'Subject: The postman | The postman |',
|
||||
'X-XML: <SpotNet><Posting><Title>The postman</Title><Description><![CDATA[post voor de lol.<br />Van "The',
|
||||
'X-XML: postman"]]></Description><Image>http://covertarget.com/si.php?id=1</Image><Category>00</Category></Posting></SpotNet>',
|
||||
'Date: 01 Feb 2010 02:43:16 GMT',
|
||||
].join('\r\n')
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw)
|
||||
assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>')
|
||||
assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT')
|
||||
const spot = parseSpotXml(xml)!
|
||||
assert.equal(spot.description, 'post voor de lol.<br />Van "The postman"')
|
||||
assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' })
|
||||
assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"')
|
||||
})
|
||||
|
||||
test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => {
|
||||
assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café')
|
||||
assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café')
|
||||
})
|
||||
|
||||
test('names categories as Spotweb does', ({ assert }) => {
|
||||
assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), {
|
||||
head: 'Image',
|
||||
subcats: [
|
||||
{ type: 'Type', values: ['Movie'] },
|
||||
{ type: 'Format', values: ['x264'] },
|
||||
{ type: 'Source', values: ['Retail'] },
|
||||
{ type: 'Language', values: ['English subtitles (available)', 'English audio/written'] },
|
||||
{ type: 'Genre', values: ['Science Fiction'] },
|
||||
],
|
||||
})
|
||||
assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] })
|
||||
})
|
||||
|
||||
test('turns UBB markup into plain text', ({ assert }) => {
|
||||
assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)')
|
||||
})
|
||||
|
||||
test('decodes an escaped, deflated NZB split over articles', ({ assert }) => {
|
||||
const nzb = '<?xml version="1.0"?>\n<nzb>\r\n=\0</nzb>'.repeat(20)
|
||||
const escaped = deflateRawSync(Buffer.from(nzb, 'latin1'))
|
||||
.toString('latin1')
|
||||
.replace(/[=\n\r\0]/g, (c) => ({ '=': '=D', '\n': '=C', '\r': '=B', '\0': '=A' })[c]!)
|
||||
// Posted as lines, dot-stuffed, over two articles.
|
||||
const lines = escaped.match(/.{1,40}/gs)!.map((l) => (l.startsWith('.') ? '.' + l : l))
|
||||
const half = Math.ceil(lines.length / 2)
|
||||
const bodies = [lines.slice(0, half), lines.slice(half)].map((ls) => Buffer.from(ls.join('\r\n'), 'latin1'))
|
||||
assert.equal(decodeSpotBinary(bodies).toString('latin1'), nzb)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('spotnet / moderation', () => {
|
||||
test('recognises delete and dispose commands from key 2 only', async ({ assert }) => {
|
||||
const { moderationTarget, spotterId } = await import('#lib/spotnet/index')
|
||||
assert.equal(moderationTarget({ keyId: 2, title: 'DISPOSE 7IzfA0aCFTkOne6agvY06@spot.net - The Voice DE S15E07 (1080p)' }), '7IzfA0aCFTkOne6agvY06@spot.net')
|
||||
assert.equal(moderationTarget({ keyId: 2, title: 'delete <abc@spot.net>' }), 'abc@spot.net')
|
||||
assert.isNull(moderationTarget({ keyId: 7, title: 'DISPOSE abc@spot.net' }))
|
||||
assert.isNull(moderationTarget({ keyId: 2, title: 'The Disposal 2024' }))
|
||||
// Spotweb: base64 of the little-endian CRC32 of the key, without / + =.
|
||||
assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('spotnet / passwords in descriptions', () => {
|
||||
test('finds passwords written in the description or title', async ({ assert }) => {
|
||||
const { passwordFromSpot } = await import('#lib/spotnet/index')
|
||||
assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2')
|
||||
assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R')
|
||||
assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813')
|
||||
assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401')
|
||||
assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo')
|
||||
assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t')
|
||||
assert.equal(passwordFromSpot('T', '<br />Password = "abc123"<br />'), 'abc123')
|
||||
})
|
||||
|
||||
test('finds passwords in the phrasings found in spots that the first version missed', async ({ assert }) => {
|
||||
const { passwordFromSpot } = await import('#lib/spotnet/index')
|
||||
// LOG.md 2026-10-08 (10)
|
||||
assert.equal(passwordFromSpot('T', 'Formaat: DVD5 ISO[br][br]!! LETOP Wachtwoord : oscfyajg !!'), 'oscfyajg')
|
||||
assert.equal(passwordFromSpot('T', "Het totaal gewicht komt neer op zo'n 400GB. Deze is voorzien van een wachtwoord. Deze is: VTSm7nLdZZl5Kren[br]"), 'VTSm7nLdZZl5Kren')
|
||||
assert.equal(passwordFromSpot('T', 'Futurama S11E10 1080P[br][br]ww in nzb: cI6f7dBV6BKkhqD[br][br]Dit was'), 'cI6f7dBV6BKkhqD')
|
||||
assert.equal(passwordFromSpot('T', '[b]Het wachtwoord voor de nzb is: dw0022[/b][br]'), 'dw0022')
|
||||
assert.equal(passwordFromSpot('T', 'haakinstructies.[br][br]Wachtwoord inbegrepen: wysiwyg'), 'wysiwyg')
|
||||
assert.equal(passwordFromSpot('T', 'afgeslagen moeten worden.[br][br]wachtwoord is arn2'), 'arn2')
|
||||
assert.equal(passwordFromSpot('T', 'Forced : No[br][br]WW:[br]tijdloos[br][br]Zag en merkte'), 'tijdloos')
|
||||
assert.equal(passwordFromSpot('T', 'telekinetische krachten beschikt.[br]WW carrie[br][br]Multisubs'), 'carrie')
|
||||
})
|
||||
|
||||
test('ignores sentences about passwords', async ({ assert }) => {
|
||||
const { passwordFromSpot } = await import('#lib/spotnet/index')
|
||||
for (const description of [
|
||||
'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]',
|
||||
'[b]--> Geen wachtwoord benodigd [/b]',
|
||||
'zitten 2 bestanden in en is met een password beschermd.',
|
||||
'It also removes the need for manual password entry and improves the reliability',
|
||||
'Codeermodus: Hoge kwaliteit (2-Pass VBR)',
|
||||
'29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper',
|
||||
'Wachtwoord: geen',
|
||||
'WW: zie nzb',
|
||||
'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!',
|
||||
'Mijn wachtwoord is hier direct te vinden, je moet het wel zelf overtypen',
|
||||
'Bringing History to Life - Greatest Battles of WW II, Sept 2026',
|
||||
'WW II',
|
||||
'WW Disc 1 = 5YIMKDaoTaKe8WIWEKUF',
|
||||
'U kunt eigenaars- en gebruikerswachtwoorden aan uw pdf toevoegen met behulp van 256-bits encryptie.',
|
||||
'APP MISC - SHAREit v6.27.49 ww 4062749 arm64 v8a.apk',
|
||||
'Het ww staat op het plaatje, type het EXACT zo in.',
|
||||
'Wachtwoord:[br]Deze download heeft er geen',
|
||||
]) {
|
||||
assert.isNull(passwordFromSpot('T', description), description)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,108 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { fileKey, parseSubject } from '#lib/subject/index'
|
||||
import { parseXrefGroups } from '#lib/nntp/xref'
|
||||
import { inferTotalParts } from '#lib/yenc/index'
|
||||
|
||||
test.group('subject / parseSubject', () => {
|
||||
test('parses file counter, filename and segment counter', ({ assert }) => {
|
||||
assert.deepEqual(parseSubject('. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc (56/274)'), {
|
||||
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
|
||||
segment: 56,
|
||||
totalSegments: 274,
|
||||
fileIndex: 9,
|
||||
fileTotal: 51,
|
||||
base: '. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc',
|
||||
})
|
||||
})
|
||||
|
||||
test('handles a file size after the segment counter', ({ assert }) => {
|
||||
const subject = '[8/9] - "x.tar.zst.vol063+064.par2" yEnc (37/72) 51315352'
|
||||
const parsed = parseSubject(subject)!
|
||||
assert.equal(parsed.segment, 37)
|
||||
assert.equal(parsed.totalSegments, 72)
|
||||
assert.equal(parsed.base, '[8/9] - "x.tar.zst.vol063+064.par2" yEnc 51315352')
|
||||
})
|
||||
|
||||
test('ignores counters inside the filename', ({ assert }) => {
|
||||
const parsed = parseSubject('"Show (1/2) Special.mkv" yEnc (3/10)')!
|
||||
assert.equal(parsed.name, 'Show (1/2) Special.mkv')
|
||||
assert.equal(parsed.segment, 3)
|
||||
assert.equal(parsed.totalSegments, 10)
|
||||
})
|
||||
|
||||
test('the base is equal for all segments of one file', ({ assert }) => {
|
||||
const a = parseSubject('[1/9] - "a.tar.zst" yEnc (253/1000) 716554110')!
|
||||
const b = parseSubject('[1/9] - "a.tar.zst" yEnc (254/1000) 716554110')!
|
||||
assert.equal(a.base, b.base)
|
||||
})
|
||||
|
||||
test('returns null for obfuscated or invalid subjects', ({ assert }) => {
|
||||
assert.isNull(parseSubject('GZFDJc0hB3Y0FwwAznkk'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc (0/5)'))
|
||||
assert.isNull(parseSubject('"name.rar" yEnc (6/5)'))
|
||||
assert.isNull(parseSubject('"" yEnc (1/5)'))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('subject / fileKey', () => {
|
||||
test('differs per poster and per segment count, is stable otherwise', ({ assert }) => {
|
||||
const key = fileKey('poster@a', '"x.rar" yEnc', 10)
|
||||
assert.lengthOf(key, 40)
|
||||
assert.equal(key, fileKey('poster@a', '"x.rar" yEnc', 10))
|
||||
assert.notEqual(key, fileKey('poster@b', '"x.rar" yEnc', 10))
|
||||
assert.notEqual(key, fileKey('poster@a', '"x.rar" yEnc', 11))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('nntp / parseXrefGroups', () => {
|
||||
test('extracts crossposted groups', ({ assert }) => {
|
||||
assert.deepEqual(
|
||||
parseXrefGroups('news.eweka.nl alt.binaries.nl:35801424104 alt.binaries.test:27275430913'),
|
||||
['alt.binaries.nl', 'alt.binaries.test']
|
||||
)
|
||||
assert.deepEqual(parseXrefGroups(undefined), [])
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / inferTotalParts', () => {
|
||||
const size = 104857600
|
||||
const partSize = 384000
|
||||
|
||||
test('uses total= when present', ({ assert }) => {
|
||||
assert.equal(inferTotalParts({ begin: { part: 2, total: 7, line: 128, size, name: 'a' } }), 7)
|
||||
})
|
||||
|
||||
test('derives the total from the part size for middle and last parts', ({ assert }) => {
|
||||
const expected = Math.ceil(size / partSize) // 274, as in the subject "(56/274)"
|
||||
assert.equal(expected, 274)
|
||||
assert.equal(
|
||||
inferTotalParts({
|
||||
begin: { part: 56, line: 128, size, name: 'a' },
|
||||
part: { begin: 21120001, end: 21504000 },
|
||||
}),
|
||||
expected
|
||||
)
|
||||
assert.equal(
|
||||
inferTotalParts({
|
||||
begin: { part: 274, line: 128, size, name: 'a' },
|
||||
part: { begin: 273 * partSize + 1, end: size },
|
||||
}),
|
||||
274
|
||||
)
|
||||
})
|
||||
|
||||
test('single-part posts have one part', ({ assert }) => {
|
||||
assert.equal(inferTotalParts({ begin: { line: 128, size: 10, name: 'a' } }), 1)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('indexer / skipBodyMessageIds', () => {
|
||||
test('matches the multi-group uploader, not other posts', async ({ assert }) => {
|
||||
const { default: indexerConfig } = await import('#config/indexer')
|
||||
const skip = (id: string) => indexerConfig.skipBodyMessageIds.some((re) => re.test(id))
|
||||
assert.isTrue(skip('<b800526$4db823e$6544066@d810c77fc0.d15fa>'))
|
||||
assert.isFalse(skip('<Part56of274.4B8CB0C9360F4548AB800FEDCB39CE07@1790532933.local>'))
|
||||
assert.isFalse(skip('<a6a525b170f44d68a756f1ba6fbd4c60@ngPost>'))
|
||||
})
|
||||
})
|
||||
@@ -1,106 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import net from 'node:net'
|
||||
import { deflateSync } from 'node:zlib'
|
||||
import { XoverClient, parseOverviews } from '#lib/nntp/xover_client'
|
||||
import { unzipSync } from 'node:zlib'
|
||||
|
||||
const LINES = [
|
||||
'101\tShow.S01E01 - "a.mkv" yEnc (1/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:00 +0000\t<one@test>\t\t700000\t5000\tXref: news.example a.b.test:101',
|
||||
'102\tShow.S01E01 - "a.mkv" yEnc (2/2)\tposter <p@x>\tMon, 05 Oct 2026 10:00:01 +0000\t<two@test>\t\t650000\t4600\tXref: news.example a.b.test:102 a.b.other:7',
|
||||
]
|
||||
|
||||
/** A tiny NNTP server: plain replies, gzip after XFEATURE, 420 for 1-1, replies split into small chunks. */
|
||||
function server(): Promise<{ port: number; close: () => void }> {
|
||||
return new Promise((resolve) => {
|
||||
const srv = net.createServer((sock) => {
|
||||
let gzip = false
|
||||
const send = (data: Buffer) => {
|
||||
for (let i = 0; i < data.length; i += 7) sock.write(data.subarray(i, i + 7))
|
||||
}
|
||||
sock.write('200 welcome\r\n')
|
||||
let pending = ''
|
||||
sock.on('data', (chunk) => {
|
||||
pending += chunk.toString()
|
||||
let at: number
|
||||
while ((at = pending.indexOf('\r\n')) !== -1) {
|
||||
const line = pending.slice(0, at)
|
||||
pending = pending.slice(at + 2)
|
||||
if (line.startsWith('AUTHINFO USER')) sock.write('381 more\r\n')
|
||||
else if (line.startsWith('AUTHINFO PASS')) sock.write('281 ok\r\n')
|
||||
else if (line === 'XFEATURE COMPRESS GZIP') (gzip = true), sock.write('290 Feature Enabled\r\n')
|
||||
else if (line.startsWith('GROUP')) sock.write('211 2 101 102 a.b.test\r\n')
|
||||
else if (line === 'XOVER 1-1') sock.write('420 No Articles Selected\r\n')
|
||||
else if (line.startsWith('XOVER')) {
|
||||
const body = Buffer.from(LINES.join('\r\n') + '\r\n')
|
||||
if (gzip) {
|
||||
sock.write('224 Overview Information Follows [COMPRESS=GZIP]\r\n')
|
||||
send(Buffer.concat([deflateSync(body), Buffer.from('.\r\n')]))
|
||||
} else {
|
||||
sock.write('224 Overview Information Follows\r\n')
|
||||
send(Buffer.concat([body, Buffer.from('.\r\n')]))
|
||||
}
|
||||
} else if (line === 'QUIT') sock.end('205 bye\r\n')
|
||||
}
|
||||
})
|
||||
})
|
||||
srv.listen(0, '127.0.0.1', () => resolve({ port: (srv.address() as net.AddressInfo).port, close: () => srv.close() }))
|
||||
})
|
||||
}
|
||||
|
||||
test.group('nntp / XoverClient', () => {
|
||||
test('parses overview lines into the fields nntp-js gives', ({ assert }) => {
|
||||
const [[number, fields]] = parseOverviews(LINES[0] + '\r\n')
|
||||
assert.equal(number, 101)
|
||||
assert.equal(fields['message-id'], '<one@test>')
|
||||
assert.equal(fields[':bytes'], '700000')
|
||||
assert.equal(fields.xref, 'news.example a.b.test:101')
|
||||
})
|
||||
|
||||
for (const compress of [false, true]) {
|
||||
test(`fetches overviews ${compress ? 'gzip-compressed' : 'plain'}, and an empty range`, async ({ assert }) => {
|
||||
const { port, close } = await server()
|
||||
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
|
||||
try {
|
||||
assert.equal(client.compressed, compress)
|
||||
assert.deepEqual(await client.group('a.b.test'), { count: 2, low: 101, high: 102 })
|
||||
const overviews = await client.xover(101, 102)
|
||||
assert.deepEqual(overviews.map(([n]) => n), [101, 102])
|
||||
assert.equal(overviews[1][1].subject, 'Show.S01E01 - "a.mkv" yEnc (2/2)')
|
||||
assert.deepEqual(await client.xover(1, 1), [])
|
||||
assert.lengthOf(await client.xover(101, 102), 2)
|
||||
} finally {
|
||||
client.close()
|
||||
close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test('idle time between commands does not count against the reply timeout', async ({ assert }) => {
|
||||
const { port, close } = await server()
|
||||
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress: true, timeoutMs: 200 })
|
||||
try {
|
||||
await client.group('a.b.test')
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
assert.lengthOf(await client.xover(101, 102), 2)
|
||||
} finally {
|
||||
client.close()
|
||||
close()
|
||||
}
|
||||
})
|
||||
|
||||
for (const compress of [false, true]) {
|
||||
test(`hands out overviews as one zlib stream (${compress ? 'as the server sent it' : 'deflated here'})`, async ({ assert }) => {
|
||||
const { port, close } = await server()
|
||||
const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress })
|
||||
try {
|
||||
await client.group('a.b.test')
|
||||
const raw = await client.xoverRaw(101, 102)
|
||||
assert.deepEqual(parseOverviews(unzipSync(raw).toString('utf8')).map(([n]) => n), [101, 102])
|
||||
assert.lengthOf(parseOverviews(unzipSync(await client.xoverRaw(1, 1)).toString('utf8')), 0)
|
||||
} finally {
|
||||
client.close()
|
||||
close()
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -1,192 +0,0 @@
|
||||
import { test } from '@japa/runner'
|
||||
import yencode from 'yencode'
|
||||
import {
|
||||
YencCrcError,
|
||||
YencError,
|
||||
YencFile,
|
||||
crc32Hex,
|
||||
decodeSegment,
|
||||
parseYencMeta,
|
||||
} from '#lib/yenc/index'
|
||||
|
||||
/** Builds an article body as a yEnc poster would send it. */
|
||||
function article(
|
||||
data: Buffer,
|
||||
opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string }
|
||||
) {
|
||||
const fileSize = opts.fileSize ?? data.length
|
||||
const encoded = yencode.encode(data).toString('latin1')
|
||||
const lines: string[] = []
|
||||
if (opts.part) {
|
||||
lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`)
|
||||
const begin = opts.begin ?? 1
|
||||
lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`)
|
||||
lines.push(encoded)
|
||||
const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : ''
|
||||
lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`)
|
||||
} else {
|
||||
lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`)
|
||||
lines.push(encoded)
|
||||
lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`)
|
||||
}
|
||||
return Buffer.from(lines.join('\r\n'), 'latin1')
|
||||
}
|
||||
|
||||
function randomBytes(length: number, seed = 1) {
|
||||
const buf = Buffer.alloc(length)
|
||||
let x = seed
|
||||
for (let i = 0; i < length; i++) {
|
||||
x = (x * 1103515245 + 12345) & 0x7fffffff
|
||||
buf[i] = x & 0xff
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
test.group('yenc / parseYencMeta', () => {
|
||||
test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => {
|
||||
const body = Buffer.from(
|
||||
'=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' +
|
||||
'=ypart begin=21120001 end=21504000\r\n' +
|
||||
'xyz\r\n' +
|
||||
'=yend size=384000 part=56 pcrc32=B6339AFC',
|
||||
'latin1'
|
||||
)
|
||||
const meta = parseYencMeta(body)
|
||||
assert.deepEqual(meta.begin, {
|
||||
part: 56,
|
||||
total: undefined,
|
||||
line: 128,
|
||||
size: 104857600,
|
||||
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
|
||||
})
|
||||
assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 })
|
||||
assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined })
|
||||
assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz')
|
||||
})
|
||||
|
||||
test('keeps spaces and "=" in the filename', ({ assert }) => {
|
||||
const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' })
|
||||
assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo')
|
||||
})
|
||||
|
||||
test('ignores leading blank lines before =ybegin', ({ assert }) => {
|
||||
const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })])
|
||||
assert.equal(parseYencMeta(body).begin.name, 'a.txt')
|
||||
})
|
||||
|
||||
test('rejects bodies without yEnc markers', ({ assert }) => {
|
||||
assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/)
|
||||
assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / decodeSegment', () => {
|
||||
test('round-trips a single-part post and checks crc32', ({ assert }) => {
|
||||
const data = randomBytes(5000)
|
||||
const segment = decodeSegment(article(data, { name: 'single.bin' }))
|
||||
assert.isTrue(segment.data.equals(data))
|
||||
assert.equal(segment.offset, 0)
|
||||
})
|
||||
|
||||
test('throws YencCrcError on a corrupted part', ({ assert }) => {
|
||||
const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 })
|
||||
const index = body.indexOf('\r\n=ypart') + 60
|
||||
body[index] = body[index] === 0x41 ? 0x42 : 0x41
|
||||
assert.throws(() => decodeSegment(body), YencCrcError as any)
|
||||
})
|
||||
|
||||
test('undoes NNTP dot-stuffing', ({ assert }) => {
|
||||
// Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..".
|
||||
const data = Buffer.from([4, 5, 6])
|
||||
const body = Buffer.from(
|
||||
`=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`,
|
||||
'latin1'
|
||||
)
|
||||
assert.isTrue(decodeSegment(body).data.equals(data))
|
||||
assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / YencFile', () => {
|
||||
test('assembles parts added out of order and verifies the file crc', ({ assert }) => {
|
||||
const file = randomBytes(10_000, 7)
|
||||
const chunk = 3000
|
||||
const total = Math.ceil(file.length / chunk)
|
||||
const bodies = Array.from({ length: total }, (_, i) =>
|
||||
article(file.subarray(i * chunk, (i + 1) * chunk), {
|
||||
name: 'multi.rar',
|
||||
fileSize: file.length,
|
||||
part: i + 1,
|
||||
total,
|
||||
begin: i * chunk + 1,
|
||||
fileCrc: i === total - 1 ? crc32Hex(file) : undefined,
|
||||
})
|
||||
)
|
||||
|
||||
const segments = bodies.map((body) => decodeSegment(body)).reverse()
|
||||
const yencFile = YencFile.fromSegment(segments[0])
|
||||
assert.equal(yencFile.totalParts, 4)
|
||||
|
||||
for (const segment of segments.slice(0, -1)) yencFile.add(segment)
|
||||
assert.isFalse(yencFile.isComplete)
|
||||
assert.deepEqual(yencFile.missingParts(), [1])
|
||||
assert.throws(() => yencFile.getBuffer(), /missing parts: 1/)
|
||||
|
||||
yencFile.add(segments.at(-1)!)
|
||||
yencFile.add(segments[0]) // duplicate is ignored
|
||||
assert.isTrue(yencFile.isComplete)
|
||||
assert.isTrue(yencFile.getBuffer().equals(file))
|
||||
})
|
||||
|
||||
test('rejects segments of a file with another size', ({ assert }) => {
|
||||
const a = decodeSegment(article(Buffer.from('aa'), { name: 'a.bin' }))
|
||||
const b = decodeSegment(article(Buffer.from('bbb'), { name: 'b.bin' }))
|
||||
assert.throws(() => YencFile.fromSegment(a).add(b), /does not belong to "a.bin"/)
|
||||
})
|
||||
|
||||
test('accepts segments with per-article random names', ({ assert }) => {
|
||||
const file = randomBytes(4000, 3)
|
||||
const [x, y] = [0, 1].map((i) =>
|
||||
decodeSegment(
|
||||
article(file.subarray(i * 2000, (i + 1) * 2000), {
|
||||
name: i === 0 ? 'Xq8r2Kd' : 'p0Lm3Zw',
|
||||
fileSize: 4000,
|
||||
part: i + 1,
|
||||
total: 2,
|
||||
begin: i * 2000 + 1,
|
||||
})
|
||||
)
|
||||
)
|
||||
const assembled = YencFile.fromSegment(x)
|
||||
assembled.add(x)
|
||||
assembled.add(y)
|
||||
assert.isTrue(assembled.getBuffer().equals(file))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('content / firstChunkMd5', () => {
|
||||
test('hashes the first 16 KiB of the decoded first part', async ({ assert }) => {
|
||||
const { firstChunkMd5 } = await import('#services/ContentService')
|
||||
const { createHash } = await import('node:crypto')
|
||||
const file = randomBytes(40_000, 5)
|
||||
const body = article(file.subarray(0, 20_000), { name: 'x.bin', fileSize: 40_000, part: 1, total: 2 })
|
||||
assert.equal(firstChunkMd5(body, 40_000), createHash('md5').update(file.subarray(0, 16384)).digest('hex'))
|
||||
// A first part shorter than 16 KiB of a larger file cannot give the fingerprint.
|
||||
const short = article(file.subarray(0, 1000), { name: 'x.bin', fileSize: 40_000, part: 1, total: 40 })
|
||||
assert.isUndefined(firstChunkMd5(short, 40_000))
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / YencFile without total=', () => {
|
||||
test('infers the part count when =ybegin has no total', ({ assert }) => {
|
||||
const file = randomBytes(5000, 11)
|
||||
const strip = (body: Buffer) => Buffer.from(body.toString('latin1').replace(/ total=\d+/, ''), 'latin1')
|
||||
const parts = [0, 1].map((i) =>
|
||||
decodeSegment(strip(article(file.subarray(i * 3000, (i + 1) * 3000), { name: 'n.nzb', fileSize: 5000, part: i + 1, total: 2, begin: i * 3000 + 1 })))
|
||||
)
|
||||
const assembled = YencFile.fromSegment(parts[0])
|
||||
assert.equal(assembled.totalParts, 2)
|
||||
parts.forEach((p) => assembled.add(p))
|
||||
assert.isTrue(assembled.getBuffer().equals(file))
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user