Daan MeijerandClaude Fable 5.1
e135076fcd
pending_ladder.ts claims the oldest waiting pending releases' next target (par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each claim; a usable name (the largest named file of a par2 set or a listing, or a lone file's yEnc name, neither packaging nor random) hands the release to the namer, no name the next target or the end of the climb; climbs count on pending_signatures per (tool, group, poster) and (tool, group), and a signature with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are discarded without a probe, the rules' sample excepted. pending_namer.ts reads a named release's articles again from the corpus or by XOVER and writes it to the catalog under the pending key (source weak-stem, the name as inner name) with its chunk and candidates. Migration 6400; config PENDING_*. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
207 lines
11 KiB
TypeScript
207 lines
11 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import { createHash } from 'node:crypto'
|
|
import { readFileSync } from 'node:fs'
|
|
import { crc32 } from 'node:zlib'
|
|
import yencode from 'yencode'
|
|
import { crc32Hex } from '#lib/yenc/index'
|
|
import { inspect, innerName, MAX_NAMES, usableName } from '#lib/probe/index'
|
|
|
|
/** A single-part yEnc article of `data`. */
|
|
function article(data: Buffer, name: string) {
|
|
const lines = [`=ybegin line=128 size=${data.length} name=${name}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} crc32=${crc32Hex(data)}`]
|
|
return Buffer.from(lines.join('\r\n'), 'latin1')
|
|
}
|
|
|
|
const md5 = (data: Buffer) => createHash('md5').update(data).digest()
|
|
/** Builds one PAR2 packet with a valid checksum. */
|
|
function packet(setId: Buffer, type: string, body: Buffer) {
|
|
const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)])
|
|
const length = Buffer.alloc(8)
|
|
length.writeBigUInt64LE(BigInt(64 + padded.length))
|
|
const typeBuf = Buffer.alloc(16)
|
|
typeBuf.write(type, 'latin1')
|
|
const hashed = Buffer.concat([setId, typeBuf, padded])
|
|
return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed])
|
|
}
|
|
function fileDesc(setId: Buffer, name: string, size: number) {
|
|
const len = Buffer.alloc(8)
|
|
len.writeBigUInt64LE(BigInt(size))
|
|
const md5_16k = md5(Buffer.from(`${name}-16k`))
|
|
const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)]))
|
|
const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')])
|
|
return packet(setId, 'PAR 2.0\0FileDesc', body)
|
|
}
|
|
const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
|
function vint(value: number) {
|
|
const bytes: number[] = []
|
|
do {
|
|
let byte = value % 128
|
|
value = Math.floor(value / 128)
|
|
if (value > 0) byte |= 0x80
|
|
bytes.push(byte)
|
|
} while (value > 0)
|
|
return Buffer.from(bytes)
|
|
}
|
|
function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) {
|
|
let flags = headerFlags
|
|
const parts = [vint(type)]
|
|
const sizes: Buffer[] = []
|
|
if (opts.extra) {
|
|
flags |= 0x0001
|
|
sizes.push(vint(opts.extra.length))
|
|
}
|
|
if (opts.data !== undefined) {
|
|
flags |= 0x0002
|
|
sizes.push(vint(opts.data))
|
|
}
|
|
const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)])
|
|
const sized = Buffer.concat([vint(header.length), header])
|
|
const crc = Buffer.alloc(4)
|
|
crc.writeUInt32LE(crc32(sized) >>> 0)
|
|
return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)])
|
|
}
|
|
function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) {
|
|
const nameBuf = Buffer.from(name, 'utf8')
|
|
const fields = Buffer.concat([
|
|
vint(opts.dir ? 0x0001 : 0), // file flags
|
|
vint(size),
|
|
vint(0x20), // attributes
|
|
vint(0), // compression info
|
|
vint(0), // host OS
|
|
vint(nameBuf.length),
|
|
nameBuf,
|
|
])
|
|
// Encryption record: size, type 1, then its data (dummy here).
|
|
const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined
|
|
return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed })
|
|
}
|
|
|
|
test.group('probe / usable name (the pending ladder)', () => {
|
|
test('a par2 or listing that names a real file names the release; random volumes and packaging do not', ({ assert }) => {
|
|
const base = { size: 1000, type: null }
|
|
assert.equal(usableName({ ...base, yencName: 'x.par2', names: ['q3B9OrpKiCxUnzgtUGnp.part01.rar', 'Some.Movie.2016.mkv'] }, 'par2'), 'Some.Movie.2016.mkv')
|
|
assert.isNull(usableName({ ...base, yencName: 'q3B9OrpKiCxUnzgtUGnp.par2', names: ['q3B9OrpKiCxUnzgtUGnp.part01.rar'] }, 'par2'))
|
|
assert.equal(usableName({ ...base, yencName: 'x.rar', names: ['Folder/Real.Name.S01E01.mkv'] }, 'archive'), 'Real.Name.S01E01.mkv')
|
|
assert.isNull(usableName({ ...base, yencName: 'x.rar', names: [], passworded: true }, 'archive'))
|
|
assert.equal(usableName({ ...base, yencName: 'brothalovers.20.02.16.hazel.moore.mp4' }, 'lone'), 'brothalovers.20.02.16.hazel.moore.mp4')
|
|
assert.isNull(usableName({ ...base, yencName: 'c3453cdd2a3645a8f3cf3d22319abe63' }, 'lone'))
|
|
assert.isNull(usableName({ ...base, yencName: 'c3453cdd2a3645a8f3cf3d22319abe63.part01.rar' }, 'lone'))
|
|
})
|
|
})
|
|
|
|
test.group('probe / inspect', () => {
|
|
test('a par2 index: its files, largest first', ({ assert }) => {
|
|
const setId = Buffer.alloc(16, 3)
|
|
const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)])
|
|
const result = inspect(article(par2, 'a8f3k2.par2'))
|
|
const { par2: set, hash16k, ...rest } = result
|
|
assert.deepEqual(rest, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
|
|
// The whole set for the par2 merge: per file the hash of its first 16 KiB, its length and name.
|
|
assert.equal(set!.setId, setId.toString('hex'))
|
|
assert.deepEqual(
|
|
set!.files.map((f) => [f.name, f.length, f.hash16k]),
|
|
[
|
|
['Movie.2020.part02.rar', 100, md5(Buffer.from('Movie.2020.part02.rar-16k')).toString('hex')],
|
|
['Movie.2020.part01.rar', 200, md5(Buffer.from('Movie.2020.part01.rar-16k')).toString('hex')],
|
|
]
|
|
)
|
|
assert.equal(hash16k, md5(par2).toString('hex'))
|
|
})
|
|
|
|
test("the hash of a file's first 16 KiB, when the article starts the file", ({ assert }) => {
|
|
const data = Buffer.alloc(40_000, 7)
|
|
assert.equal(inspect(article(data, 'x')).hash16k, md5(data.subarray(0, 16384)).toString('hex'))
|
|
const part = Buffer.from(
|
|
[`=ybegin part=2 total=3 line=128 size=120000 name=x`, `=ypart begin=40001 end=80000`, yencode.encode(data).toString('latin1'), `=yend size=40000 part=2 pcrc32=${crc32Hex(data)}`].join('\r\n'),
|
|
'latin1'
|
|
)
|
|
assert.isUndefined(inspect(part).hash16k)
|
|
})
|
|
|
|
test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => {
|
|
const rar = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('Movie.2020.1080p/movie.mkv', 5_000_000, { packed: 10 }), rar5File('Movie.2020.1080p/sample.mkv', 1000, { packed: 10 })])
|
|
const open = inspect(article(rar, 'x7q.part01.rar'))
|
|
assert.include(open, { type: 'rar5', entries: 2, passworded: false })
|
|
assert.deepEqual(open.names, ['Movie.2020.1080p/movie.mkv', 'Movie.2020.1080p/sample.mkv'])
|
|
const locked = inspect(article(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]), 'y.rar'))
|
|
assert.isTrue(locked.passworded)
|
|
const hidden = inspect(article(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]), 'z.rar'))
|
|
assert.include(hidden, { passworded: true, entries: 0 })
|
|
})
|
|
|
|
test('other data: the yEnc name and what the bytes are', ({ assert }) => {
|
|
const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)])
|
|
assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska', hash16k: md5(mkv).toString('hex') })
|
|
})
|
|
|
|
test('not yEnc: throws', ({ assert }) => {
|
|
assert.throws(() => inspect(Buffer.from('just some text\r\n')))
|
|
})
|
|
|
|
test('at most MAX_NAMES names', ({ assert }) => {
|
|
const setId = Buffer.alloc(16, 4)
|
|
const par2 = Buffer.concat(Array.from({ length: MAX_NAMES + 5 }, (_, i) => fileDesc(setId, `f${i}.rar`, i)))
|
|
const result = inspect(article(par2, 'p.par2'))
|
|
assert.equal(result.entries, MAX_NAMES + 5)
|
|
assert.lengthOf(result.names!, MAX_NAMES)
|
|
})
|
|
})
|
|
|
|
test.group('probe / innerName', () => {
|
|
test("the archive's largest entry, else par2's largest non-packaging file, else a lone file's yEnc name", ({ assert }) => {
|
|
const archive = { yencName: 'a.rar', size: 1, type: 'rar5' as const, names: ['Show.S01E01/show.s01e01.mkv'] }
|
|
const par2 = { yencName: 'a.par2', size: 1, type: 'par2' as const, names: ['abc.part01.rar', 'Show.S01E01.1080p.mkv'] }
|
|
assert.equal(innerName([{ kind: 'par2', result: par2 }, { kind: 'archive', result: archive }]), 'show.s01e01.mkv')
|
|
assert.equal(innerName([{ kind: 'par2', result: par2 }]), 'Show.S01E01.1080p.mkv')
|
|
assert.equal(innerName([{ kind: 'par2', result: { ...par2, names: ['abc.part01.rar', 'abc.par2'] } }]), null)
|
|
const lone = { kind: 'lone', result: { yencName: 'Real.Name.mkv', size: 1, type: 'matroska' as const } }
|
|
assert.equal(innerName([lone], 'k3j4h5'), 'Real.Name.mkv')
|
|
// A random-named file whose yEnc name is its random name says nothing new.
|
|
assert.isNull(innerName([lone], 'real.name.mkv'))
|
|
assert.isNull(innerName([lone], 'Real.Name'))
|
|
// Nor does packaging: a lone par2 volume.
|
|
assert.isNull(innerName([{ kind: 'lone', result: { yencName: 'a1b2.vol007+08.par2', size: 1, type: 'par2' } }], 'x9y8'))
|
|
})
|
|
|
|
test("a 7z-tail: the listing from the last segment of the last volume; one that isn't the last segment: none", ({ assert }) => {
|
|
const tail = readFileSync(new URL('../../fixtures/sevenzip/split-last-volume.tail', import.meta.url))
|
|
// The volume's last 8,000 bytes as its third part, of a volume 1,000,000 bytes long.
|
|
const last = tail.subarray(tail.length - 8000)
|
|
const part = (data: Buffer, begin: number) =>
|
|
Buffer.from(
|
|
[`=ybegin part=3 total=3 line=128 size=1000000 name=x.7z.004`, `=ypart begin=${begin} end=${begin + data.length - 1}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} part=3 pcrc32=${crc32Hex(data)}`].join('\r\n'),
|
|
'latin1'
|
|
)
|
|
const result = inspect(part(last, 1_000_000 - 8000 + 1), '7z-tail')
|
|
assert.equal(result.entries, 4)
|
|
assert.deepEqual(result.names?.[0], 'Some.Movie.2020.1080p.BluRay.x264-GRP/some.movie.2020.1080p.bluray.x264-grp.mkv')
|
|
assert.isFalse(result.passworded)
|
|
assert.isUndefined(inspect(part(last, 1), '7z-tail').names)
|
|
const encrypted = readFileSync(new URL('../../fixtures/sevenzip/header-encrypted.tail', import.meta.url))
|
|
const hidden = inspect(part(encrypted.subarray(encrypted.length - 8000), 1_000_000 - 8000 + 1), '7z-tail')
|
|
assert.isTrue(hidden.passworded)
|
|
assert.equal(hidden.entries, 0)
|
|
assert.isUndefined(hidden.names)
|
|
// A small 7z whole in one article, probed as an archive start.
|
|
assert.deepEqual(inspect(article(readFileSync(new URL('../../fixtures/sevenzip/tiny.7z', import.meta.url)), 'x.7z'), 'archive').names, ['b.nfo', 'a.txt'])
|
|
})
|
|
|
|
test('a ZIP start: the names of its first entries, and whether they are encrypted', ({ assert }) => {
|
|
const local = (name: string, data: Buffer, flags = 0) => {
|
|
const header = Buffer.alloc(30)
|
|
header.writeUInt32LE(0x04034b50, 0)
|
|
header.writeUInt16LE(flags, 6)
|
|
header.writeUInt32LE(data.length, 18)
|
|
header.writeUInt32LE(data.length, 22)
|
|
header.writeUInt16LE(Buffer.byteLength(name), 26)
|
|
return Buffer.concat([header, Buffer.from(name), data])
|
|
}
|
|
const zip = Buffer.concat([local('Album/', Buffer.alloc(0)), local('Album/01 - Track.flac', Buffer.alloc(3000, 1)), local('Album/cover.jpg', Buffer.alloc(500, 2))])
|
|
const plain = inspect(article(zip, 'album.zip'))
|
|
assert.equal(plain.type, 'zip')
|
|
assert.deepEqual(plain.names, ['Album/01 - Track.flac', 'Album/cover.jpg'])
|
|
assert.isFalse(plain.passworded)
|
|
assert.isTrue(inspect(article(local('secret.mkv', Buffer.alloc(100), 0x1), 'x.zip')).passworded)
|
|
})
|
|
})
|