Rebuild: phase 2's prober reads the probe queue
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d72fe2f13f
commit
8bd6d6aa5f
30 files changed
+1666
-7
No files matched your search
@@ -0,0 +1,60 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
|
||||
const md5 = (data: Buffer) => createHash('md5').update(data).digest()
|
||||
|
||||
/** Builds one PAR2 packet with a valid checksum. */
|
||||
function packet(setId: Buffer, type: string, body: Buffer) {
|
||||
const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
|
||||
const length = Buffer.alloc(8)
|
||||
length.writeBigUInt64LE(BigInt(64 + padded.length))
|
||||
const typeBuf = Buffer.alloc(16)
|
||||
typeBuf.write(type, 'latin1')
|
||||
const hashed = Buffer.concat([setId, typeBuf, padded])
|
||||
return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
|
||||
}
|
||||
|
||||
function fileDesc(setId: Buffer, name: string, size: number) {
|
||||
const len = Buffer.alloc(8)
|
||||
len.writeBigUInt64LE(BigInt(size))
|
||||
const md5_16k = md5(Buffer.from(`${name}-16k`))
|
||||
const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
|
||||
const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
|
||||
return packet(setId, 'PAR 2.0\0FileDesc', body)
|
||||
}
|
||||
|
||||
test.group('par2 / parsePar2', () => {
|
||||
const setId = Buffer.alloc(16, 7)
|
||||
|
||||
test('reads file names and sizes from FileDesc packets', ({ assert }) => {
|
||||
const data = Buffer.concat([
|
||||
packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)),
|
||||
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600),
|
||||
fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800),
|
||||
fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common
|
||||
])
|
||||
assert.isTrue(isPar2(data))
|
||||
const set = parsePar2(data)!
|
||||
assert.equal(set.recoverySetId, setId.toString('hex'))
|
||||
assert.deepEqual(
|
||||
set.files.map((f) => [f.name, f.length]),
|
||||
[
|
||||
['Some.Movie.2024.part01.rar', 104857600],
|
||||
['Some.Movie.2024.part02.rar', 52428800],
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
test('skips corrupted packets and resumes at the next one', ({ assert }) => {
|
||||
const bad = fileDesc(setId, 'bad.bin', 1)
|
||||
bad[100] ^= 0xff
|
||||
const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)])
|
||||
assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin'])
|
||||
})
|
||||
|
||||
test('returns null for non-par2 data', ({ assert }) => {
|
||||
assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1')))
|
||||
assert.isNull(parsePar2(Buffer.from('hello world')))
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user