Rebuild: phase 2's prober reads the probe queue
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d72fe2f13f
commit
8bd6d6aa5f
30 files changed
+1666
-7
No files matched your search
@@ -0,0 +1,107 @@
|
||||
import { test } from '@japa/runner'
|
||||
import yencode from 'yencode'
|
||||
import {
|
||||
YencCrcError,
|
||||
YencError,
|
||||
crc32Hex,
|
||||
decodeSegment,
|
||||
parseYencMeta,
|
||||
} from '#lib/yenc/index'
|
||||
|
||||
/** Builds an article body as a yEnc poster would send it. */
|
||||
function article(
|
||||
data: Buffer,
|
||||
opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string }
|
||||
) {
|
||||
const fileSize = opts.fileSize ?? data.length
|
||||
const encoded = yencode.encode(data).toString('latin1')
|
||||
const lines: string[] = []
|
||||
if (opts.part) {
|
||||
lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`)
|
||||
const begin = opts.begin ?? 1
|
||||
lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`)
|
||||
lines.push(encoded)
|
||||
const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : ''
|
||||
lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`)
|
||||
} else {
|
||||
lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`)
|
||||
lines.push(encoded)
|
||||
lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`)
|
||||
}
|
||||
return Buffer.from(lines.join('\r\n'), 'latin1')
|
||||
}
|
||||
|
||||
function randomBytes(length: number, seed = 1) {
|
||||
const buf = Buffer.alloc(length)
|
||||
let x = seed
|
||||
for (let i = 0; i < length; i++) {
|
||||
x = (x * 1103515245 + 12345) & 0x7fffffff
|
||||
buf[i] = x & 0xff
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
test.group('yenc / parseYencMeta', () => {
|
||||
test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => {
|
||||
const body = Buffer.from(
|
||||
'=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' +
|
||||
'=ypart begin=21120001 end=21504000\r\n' +
|
||||
'xyz\r\n' +
|
||||
'=yend size=384000 part=56 pcrc32=B6339AFC',
|
||||
'latin1'
|
||||
)
|
||||
const meta = parseYencMeta(body)
|
||||
assert.deepEqual(meta.begin, {
|
||||
part: 56,
|
||||
total: undefined,
|
||||
line: 128,
|
||||
size: 104857600,
|
||||
name: 'obrMkrDeLmbhaouwYO34.part08.rar',
|
||||
})
|
||||
assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 })
|
||||
assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined })
|
||||
assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz')
|
||||
})
|
||||
|
||||
test('keeps spaces and "=" in the filename', ({ assert }) => {
|
||||
const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' })
|
||||
assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo')
|
||||
})
|
||||
|
||||
test('ignores leading blank lines before =ybegin', ({ assert }) => {
|
||||
const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })])
|
||||
assert.equal(parseYencMeta(body).begin.name, 'a.txt')
|
||||
})
|
||||
|
||||
test('rejects bodies without yEnc markers', ({ assert }) => {
|
||||
assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/)
|
||||
assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/)
|
||||
})
|
||||
})
|
||||
|
||||
test.group('yenc / decodeSegment', () => {
|
||||
test('round-trips a single-part post and checks crc32', ({ assert }) => {
|
||||
const data = randomBytes(5000)
|
||||
const segment = decodeSegment(article(data, { name: 'single.bin' }))
|
||||
assert.isTrue(segment.data.equals(data))
|
||||
assert.equal(segment.offset, 0)
|
||||
})
|
||||
|
||||
test('throws YencCrcError on a corrupted part', ({ assert }) => {
|
||||
const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 })
|
||||
const index = body.indexOf('\r\n=ypart') + 60
|
||||
body[index] = body[index] === 0x41 ? 0x42 : 0x41
|
||||
assert.throws(() => decodeSegment(body), YencCrcError as any)
|
||||
})
|
||||
|
||||
test('undoes NNTP dot-stuffing', ({ assert }) => {
|
||||
// Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..".
|
||||
const data = Buffer.from([4, 5, 6])
|
||||
const body = Buffer.from(
|
||||
`=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`,
|
||||
'latin1'
|
||||
)
|
||||
assert.isTrue(decodeSegment(body).data.equals(data))
|
||||
assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any)
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user