implemented a lot of features
This commit is contained in:
Daan Meijer committed
1 parent
62f1767df3
commit
bcb9be5da9
141 files changed
+10705
-385
No files matched your search
@@ -0,0 +1,4 @@
|
||||
/**
|
||||
* Content recognition by magic bytes, independent of AdonisJS.
|
||||
*/
|
||||
export { sniffContentType, type ContentType } from './sniff.js'
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Recognises a file type from its first bytes, independent of AdonisJS.
|
||||
* `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
|
||||
* "nzbe", 16 random bytes (salt/IV), then ciphertext.
|
||||
*/
|
||||
export type ContentType =
|
||||
| 'nzbe'
|
||||
| 'par2'
|
||||
| 'rar4'
|
||||
| 'rar5'
|
||||
| '7z'
|
||||
| 'zip'
|
||||
| 'zstd'
|
||||
| 'gzip'
|
||||
| 'xz'
|
||||
| 'matroska'
|
||||
| 'mp4'
|
||||
| 'rclone'
|
||||
| 'iso'
|
||||
| 'udf'
|
||||
| 'pdf'
|
||||
| 'nzb'
|
||||
| 'jpeg'
|
||||
| 'png'
|
||||
| 'mp3'
|
||||
| 'flac'
|
||||
| 'avi'
|
||||
| 'wav'
|
||||
| 'ogg'
|
||||
| 'bzip2'
|
||||
|
||||
const SIGNATURES: [ContentType, number, number[]][] = [
|
||||
['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
|
||||
['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
|
||||
['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
|
||||
['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]],
|
||||
['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]],
|
||||
['zip', 0, [0x50, 0x4b, 0x03, 0x04]],
|
||||
['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]],
|
||||
['gzip', 0, [0x1f, 0x8b]],
|
||||
['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]],
|
||||
['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]],
|
||||
['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box
|
||||
['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt
|
||||
['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]],
|
||||
['jpeg', 0, [0xff, 0xd8, 0xff]],
|
||||
['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]],
|
||||
['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag
|
||||
['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC"
|
||||
['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS"
|
||||
['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh"
|
||||
]
|
||||
|
||||
export function sniffContentType(data: Buffer): ContentType | null {
|
||||
for (const [type, offset, bytes] of SIGNATURES) {
|
||||
if (data.length >= offset + bytes.length && bytes.every((b, i) => data[offset + i] === b)) return type
|
||||
}
|
||||
// Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
|
||||
// ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.
|
||||
if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso'
|
||||
if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' &&
|
||||
/^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf'
|
||||
// RIFF container: AVI or WAV by form type.
|
||||
if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') {
|
||||
const form = data.subarray(8, 12).toString('latin1')
|
||||
if (form === 'AVI ') return 'avi'
|
||||
if (form === 'WAVE') return 'wav'
|
||||
}
|
||||
const head = data.subarray(0, 512).toString('latin1')
|
||||
if (/^\s*(<\?xml[^>]*>\s*)?(<!DOCTYPE nzb|<nzb[\s>])/.test(head)) return 'nzb'
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/** Newznab categories this indexer assigns (see release_category() in SQL). */
|
||||
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
|
||||
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
|
||||
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
|
||||
]
|
||||
|
||||
/** Parent category of a (sub)category, e.g. 5040 → 5000. */
|
||||
export function parentCategory(id: number): number {
|
||||
return Math.floor(id / 1000) * 1000
|
||||
}
|
||||
|
||||
/** "TV > HD" style label. */
|
||||
export function categoryLabel(id: number): string {
|
||||
const parent = CATEGORIES.find((c) => c.id === parentCategory(id))
|
||||
if (!parent) return 'Other'
|
||||
const sub = parent.subcats.find((s) => s.id === id)
|
||||
return sub ? `${parent.name} > ${sub.name}` : parent.name
|
||||
}
|
||||
|
||||
/**
|
||||
* Expands requested categories (`cat=5000,2040`) to the concrete ones to
|
||||
* match: a parent includes all of its subcategories.
|
||||
*/
|
||||
export function expandCategories(requested: number[]): number[] {
|
||||
const result = new Set<number>()
|
||||
for (const id of requested) {
|
||||
const parent = CATEGORIES.find((c) => c.id === id)
|
||||
if (parent) {
|
||||
result.add(parent.id)
|
||||
for (const sub of parent.subcats) result.add(sub.id)
|
||||
} else {
|
||||
result.add(id)
|
||||
}
|
||||
}
|
||||
return [...result]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Newznab API formatting, independent of AdonisJS. See CONCEPT.md (D23).
|
||||
*/
|
||||
export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js'
|
||||
export { capsXml, errorXml, resultsXml, type NewznabItem, type NewznabServer } from './xml.js'
|
||||
@@ -0,0 +1,95 @@
|
||||
import { create } from 'xmlbuilder2'
|
||||
import { CATEGORIES, categoryLabel, parentCategory } from './categories.js'
|
||||
|
||||
const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/'
|
||||
|
||||
export interface NewznabItem {
|
||||
id: number
|
||||
title: string
|
||||
category: number
|
||||
size: number
|
||||
files: number
|
||||
postedAt: Date | null
|
||||
poster: string | null
|
||||
groups: string[]
|
||||
password: boolean
|
||||
season?: string
|
||||
episode?: string
|
||||
}
|
||||
|
||||
export interface NewznabServer {
|
||||
title: string
|
||||
/** Absolute base URL, e.g. http://homelab:3333 */
|
||||
baseUrl: string
|
||||
/** Appended to download links so a client can fetch without extra config. */
|
||||
apiKey?: string
|
||||
}
|
||||
|
||||
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
|
||||
export function errorXml(code: number, description: string): string {
|
||||
return create({ version: '1.0', encoding: 'UTF-8' })
|
||||
.ele('error', { code: String(code), description })
|
||||
.end({ prettyPrint: true })
|
||||
}
|
||||
|
||||
export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string {
|
||||
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
|
||||
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
|
||||
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
|
||||
caps.ele('registration', { available: 'no', open: 'no' })
|
||||
const searching = caps.ele('searching')
|
||||
searching.ele('search', { available: 'yes', supportedParams: 'q' })
|
||||
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
|
||||
searching.ele('movie-search', { available: 'yes', supportedParams: 'q' })
|
||||
searching.ele('audio-search', { available: 'yes', supportedParams: 'q' })
|
||||
searching.ele('book-search', { available: 'yes', supportedParams: 'q' })
|
||||
const categories = caps.ele('categories')
|
||||
for (const category of CATEGORIES) {
|
||||
const element = categories.ele('category', { id: String(category.id), name: category.name })
|
||||
for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name })
|
||||
}
|
||||
return caps.end({ prettyPrint: true })
|
||||
}
|
||||
|
||||
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string {
|
||||
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
|
||||
'version': '2.0',
|
||||
'xmlns:atom': 'http://www.w3.org/2005/Atom',
|
||||
'xmlns:newznab': NEWZNAB_NS,
|
||||
})
|
||||
const channel = rss.ele('channel')
|
||||
channel.ele('title').txt(server.title)
|
||||
channel.ele('description').txt(`${server.title} API results`)
|
||||
channel.ele('link').txt(`${server.baseUrl}/`)
|
||||
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
|
||||
|
||||
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
|
||||
for (const item of items) {
|
||||
const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}`
|
||||
const pubDate = (item.postedAt ?? new Date()).toUTCString()
|
||||
const element = channel.ele('item')
|
||||
element.ele('title').txt(item.title)
|
||||
element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id))
|
||||
element.ele('link').txt(getUrl)
|
||||
element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
|
||||
element.ele('pubDate').txt(pubDate)
|
||||
element.ele('category').txt(categoryLabel(item.category))
|
||||
element.ele('description').txt(item.title)
|
||||
element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
|
||||
|
||||
const attr = (name: string, value: string | number) =>
|
||||
element.ele('newznab:attr', { name, value: String(value) })
|
||||
attr('category', parentCategory(item.category))
|
||||
if (item.category !== parentCategory(item.category)) attr('category', item.category)
|
||||
attr('size', item.size)
|
||||
attr('files', item.files)
|
||||
attr('grabs', 0)
|
||||
attr('password', item.password ? 1 : 0)
|
||||
if (item.poster) attr('poster', item.poster)
|
||||
if (item.groups.length) attr('group', item.groups[0])
|
||||
attr('usenetdate', pubDate)
|
||||
if (item.season) attr('season', item.season)
|
||||
if (item.episode) attr('episode', item.episode)
|
||||
}
|
||||
return rss.end({ prettyPrint: true })
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
export interface GroupInfo {
|
||||
count: bigint
|
||||
first: bigint
|
||||
last: bigint
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a `211 count first last name` GROUP reply (RFC 3977 §6.1.1).
|
||||
*
|
||||
* nntp-js's own `group()` destructures this reply without skipping the status
|
||||
* code, so its `first`/`last` are off by one field. Pass it `result.resp`.
|
||||
*/
|
||||
export function parseGroupResponse(resp: string): GroupInfo {
|
||||
const match = /^211\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)/.exec(resp.trim())
|
||||
if (!match) throw new Error(`Unexpected GROUP response: ${resp}`)
|
||||
return {
|
||||
count: BigInt(match[1]),
|
||||
first: BigInt(match[2]),
|
||||
last: BigInt(match[3]),
|
||||
name: match[4],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Returns the group names from an Xref header,
|
||||
* e.g. `news.eweka.nl alt.binaries.test:123 alt.binaries.x:456`.
|
||||
*/
|
||||
export function parseXrefGroups(xref: string | undefined): string[] {
|
||||
if (!xref) return []
|
||||
return xref
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.slice(1)
|
||||
.map((entry) => entry.slice(0, entry.lastIndexOf(':')))
|
||||
.filter(Boolean)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { create } from 'xmlbuilder2'
|
||||
|
||||
export interface NzbFile {
|
||||
/** Name the downloader should use; the real name when it is known. */
|
||||
name: string
|
||||
poster: string
|
||||
/** Posting time in milliseconds. */
|
||||
date: number
|
||||
groups: string[]
|
||||
/** Segment number → message-id (without angle brackets) and article size. */
|
||||
segments: Record<string, { id: string; size: number }>
|
||||
}
|
||||
|
||||
export interface NzbOptions {
|
||||
/** Shown by downloaders as the job name. */
|
||||
title?: string
|
||||
/** Archive password, when known. */
|
||||
password?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an NZB 1.1 document (newzbin DTD). The subject follows the common
|
||||
* `"name" yEnc (1/N)` form, which downloaders use to name the file.
|
||||
*/
|
||||
export function buildNzb(files: NzbFile[], options: NzbOptions = {}): string {
|
||||
const root = create({ version: '1.0', encoding: 'UTF-8' })
|
||||
.dtd({ pubID: '-//newzBin//DTD NZB 1.1//EN', sysID: 'http://www.newzbin.com/DTD/nzb/nzb-1.1.dtd' })
|
||||
.ele('nzb', { xmlns: 'http://www.newzbin.com/DTD/2003/nzb' })
|
||||
|
||||
if (options.title || options.password) {
|
||||
const head = root.ele('head')
|
||||
if (options.title) head.ele('meta', { type: 'name' }).txt(options.title)
|
||||
if (options.password) head.ele('meta', { type: 'password' }).txt(options.password)
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const numbers = Object.keys(file.segments)
|
||||
.map(Number)
|
||||
.sort((a, b) => a - b)
|
||||
const element = root.ele('file', {
|
||||
poster: file.poster,
|
||||
date: String(Math.floor(file.date / 1000)),
|
||||
subject: `"${file.name}" yEnc (1/${numbers.length})`,
|
||||
})
|
||||
const groups = element.ele('groups')
|
||||
for (const group of file.groups) groups.ele('group').txt(group)
|
||||
const segments = element.ele('segments')
|
||||
for (const number of numbers) {
|
||||
const segment = file.segments[String(number)]
|
||||
segments.ele('segment', { bytes: String(segment.size), number: String(number) }).txt(segment.id)
|
||||
}
|
||||
}
|
||||
return root.end({ prettyPrint: true })
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* NZB generation, independent of AdonisJS.
|
||||
*/
|
||||
export { buildNzb, type NzbFile, type NzbOptions } from './build.js'
|
||||
export { parseNzb, type ParsedNzb, type ParsedNzbFile } from './parse.js'
|
||||
@@ -0,0 +1,72 @@
|
||||
export interface ParsedNzbFile {
|
||||
poster: string
|
||||
/** Posting time in seconds (as in the NZB). */
|
||||
date: number
|
||||
subject: string
|
||||
groups: string[]
|
||||
segments: { number: number; bytes: number; id: string }[]
|
||||
}
|
||||
|
||||
export interface ParsedNzb {
|
||||
/** <head><meta type="..."> values, e.g. name, password, category. */
|
||||
meta: Record<string, string>
|
||||
files: ParsedNzbFile[]
|
||||
}
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
|
||||
|
||||
function decode(text: string): string {
|
||||
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
if (entity[0] === '#') {
|
||||
const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
|
||||
}
|
||||
return ENTITIES[entity.toLowerCase()] ?? match
|
||||
})
|
||||
}
|
||||
|
||||
function attributes(tag: string): Record<string, string> {
|
||||
const result: Record<string, string> = {}
|
||||
for (const m of tag.matchAll(/([\w:-]+)\s*=\s*("([^"]*)"|'([^']*)')/g)) result[m[1]] = decode(m[3] ?? m[4] ?? '')
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses an NZB (1.0/1.1 and the older 0.9 DTD). A tolerant scanner rather
|
||||
* than a full XML parser: NZBs are flat, and this never resolves DTDs or
|
||||
* external entities. Returns null when there is no <nzb> root.
|
||||
*/
|
||||
export function parseNzb(xml: string, limits = { maxFiles: 20_000 }): ParsedNzb | null {
|
||||
if (!/<nzb[\s>]/i.test(xml)) return null
|
||||
|
||||
const meta: Record<string, string> = {}
|
||||
const head = /<head>([\s\S]*?)<\/head>/i.exec(xml)?.[1] ?? ''
|
||||
for (const m of head.matchAll(/<meta\s([^>]*)>([\s\S]*?)<\/meta>/gi)) {
|
||||
const type = attributes(m[1]).type
|
||||
if (type) meta[type.toLowerCase()] = decode(m[2].trim())
|
||||
}
|
||||
|
||||
const files: ParsedNzbFile[] = []
|
||||
for (const m of xml.matchAll(/<file\s([^>]*)>([\s\S]*?)<\/file>/gi)) {
|
||||
if (files.length >= limits.maxFiles) break
|
||||
const attrs = attributes(m[1])
|
||||
const body = m[2]
|
||||
const groups = [...body.matchAll(/<group>([^<]*)<\/group>/gi)].map((g) => decode(g[1].trim())).filter(Boolean)
|
||||
const segments = [...body.matchAll(/<segment\s([^>]*)>([^<]*)<\/segment>/gi)]
|
||||
.map((s) => {
|
||||
const a = attributes(s[1])
|
||||
return { number: Number(a.number), bytes: Number(a.bytes) || 0, id: decode(s[2].trim()).replace(/^<|>$/g, '') }
|
||||
})
|
||||
.filter((s) => Number.isInteger(s.number) && s.number > 0 && s.id)
|
||||
.sort((x, y) => x.number - y.number)
|
||||
if (segments.length === 0) continue
|
||||
files.push({
|
||||
poster: attrs.poster ?? '',
|
||||
date: Number(attrs.date) || 0,
|
||||
subject: attrs.subject ?? '',
|
||||
groups,
|
||||
segments,
|
||||
})
|
||||
}
|
||||
return { meta, files }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* PAR2 parsing, independent of AdonisJS. Used to recover the real filenames
|
||||
* and sizes of obfuscated posts. See CONCEPT.md (D16).
|
||||
*/
|
||||
export { parsePar2, isPar2, type Par2File, type Par2Set } from './parse.js'
|
||||
@@ -0,0 +1,74 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
export interface Par2File {
|
||||
/** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */
|
||||
fileId: string
|
||||
/** MD5 of the whole file (hex). */
|
||||
md5: string
|
||||
/** MD5 of the first 16 KiB (hex). */
|
||||
md5_16k: string
|
||||
length: number
|
||||
name: string
|
||||
}
|
||||
|
||||
export interface Par2Set {
|
||||
/** Recovery set ID (hex), shared by every packet of one par2 set. */
|
||||
recoverySetId: string
|
||||
files: Par2File[]
|
||||
}
|
||||
|
||||
const MAGIC = Buffer.from('PAR2\0PKT', 'latin1')
|
||||
const HEADER_SIZE = 64
|
||||
const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1')
|
||||
|
||||
export function isPar2(data: Buffer): boolean {
|
||||
return data.subarray(0, MAGIC.length).equals(MAGIC)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the FileDesc packets of a par2 file (index or recovery volume).
|
||||
* Packets with a bad length or checksum are skipped; parsing resumes at the
|
||||
* next magic sequence. Returns null when no valid FileDesc packet is found.
|
||||
*/
|
||||
export function parsePar2(data: Buffer): Par2Set | null {
|
||||
const files = new Map<string, Par2File>()
|
||||
let recoverySetId: string | undefined
|
||||
let offset = data.indexOf(MAGIC)
|
||||
|
||||
while (offset !== -1 && offset + HEADER_SIZE <= data.length) {
|
||||
const length = Number(data.readBigUInt64LE(offset + 8))
|
||||
const valid =
|
||||
length >= HEADER_SIZE &&
|
||||
length % 4 === 0 &&
|
||||
offset + length <= data.length &&
|
||||
createHash('md5')
|
||||
.update(data.subarray(offset + 32, offset + length))
|
||||
.digest()
|
||||
.equals(data.subarray(offset + 16, offset + 32))
|
||||
|
||||
if (!valid) {
|
||||
offset = data.indexOf(MAGIC, offset + 1)
|
||||
continue
|
||||
}
|
||||
|
||||
const setId = data.subarray(offset + 32, offset + 48).toString('hex')
|
||||
const type = data.subarray(offset + 48, offset + 64)
|
||||
if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) {
|
||||
recoverySetId = setId
|
||||
const body = data.subarray(offset + HEADER_SIZE, offset + length)
|
||||
const fileId = body.subarray(0, 16).toString('hex')
|
||||
files.set(fileId, {
|
||||
fileId,
|
||||
md5: body.subarray(16, 32).toString('hex'),
|
||||
md5_16k: body.subarray(32, 48).toString('hex'),
|
||||
length: Number(body.readBigUInt64LE(48)),
|
||||
// The name is padded with NULs to a multiple of 4 bytes.
|
||||
name: body.subarray(56).toString('utf8').replace(/\0+$/, ''),
|
||||
})
|
||||
}
|
||||
offset += length
|
||||
}
|
||||
|
||||
if (!recoverySetId || files.size === 0) return null
|
||||
return { recoverySetId, files: [...files.values()] }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* RAR (4.x and 5.0) header parsing, independent of AdonisJS. Lists archive
|
||||
* contents from the first segment of a first volume. See CONCEPT.md (D19).
|
||||
*/
|
||||
export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
|
||||
@@ -0,0 +1,333 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
export interface RarEntry {
|
||||
name: string
|
||||
/** Unpacked size in bytes (null when the archiver did not record it). */
|
||||
size: number | null
|
||||
packedSize: number
|
||||
directory: boolean
|
||||
/** The file's data is password protected. */
|
||||
encrypted: boolean
|
||||
/** Continues from the previous volume. */
|
||||
splitBefore: boolean
|
||||
/** Continues in the next volume. */
|
||||
splitAfter: boolean
|
||||
/** Stored without compression, so the data can be copied out as is. */
|
||||
stored: boolean
|
||||
/** Offset of the entry's (packed) data in the parsed buffer. */
|
||||
dataOffset: number
|
||||
}
|
||||
|
||||
export interface RarInfo {
|
||||
format: 'rar4' | 'rar5'
|
||||
/** Part of a multi-volume set. */
|
||||
volume: boolean
|
||||
/** First volume of the set (null when the format version does not say). */
|
||||
firstVolume: boolean | null
|
||||
solid: boolean
|
||||
/** File names are encrypted too; nothing can be listed without the password. */
|
||||
headersEncrypted: boolean
|
||||
entries: RarEntry[]
|
||||
/** The buffer ended before the end-of-archive header (normal for one segment). */
|
||||
truncated: boolean
|
||||
}
|
||||
|
||||
const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
|
||||
const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
||||
|
||||
export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
|
||||
if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5'
|
||||
if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4'
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists a RAR archive from its headers only, so the first segment of a first
|
||||
* volume is enough. Returns null when the data is not a RAR archive.
|
||||
*/
|
||||
export function parseRarHeaders(data: Buffer): RarInfo | null {
|
||||
const format = rarFormat(data)
|
||||
if (format === 'rar4') return parseRar4(data)
|
||||
if (format === 'rar5') return parseRar5(data)
|
||||
return null
|
||||
}
|
||||
|
||||
// --- RAR 4.x (RAR 1.5 – 4.x archive format) -------------------------------
|
||||
|
||||
const R4_MAIN = 0x73
|
||||
const R4_FILE = 0x74
|
||||
const R4_END = 0x7b
|
||||
const R4_LONG_BLOCK = 0x8000
|
||||
|
||||
function parseRar4(data: Buffer): RarInfo {
|
||||
const info: RarInfo = {
|
||||
format: 'rar4',
|
||||
volume: false,
|
||||
firstVolume: null,
|
||||
solid: false,
|
||||
headersEncrypted: false,
|
||||
entries: [],
|
||||
truncated: true,
|
||||
}
|
||||
let p = RAR4_MARKER.length
|
||||
|
||||
while (p + 7 <= data.length) {
|
||||
const headCrc = data.readUInt16LE(p)
|
||||
const type = data[p + 2]
|
||||
const flags = data.readUInt16LE(p + 3)
|
||||
const size = data.readUInt16LE(p + 5)
|
||||
if (size < 7 || p + size > data.length) break
|
||||
|
||||
if (type === R4_END) {
|
||||
info.truncated = false
|
||||
break
|
||||
}
|
||||
|
||||
let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0)
|
||||
|
||||
if (type === R4_MAIN) {
|
||||
info.volume = Boolean(flags & 0x0001)
|
||||
info.solid = Boolean(flags & 0x0008)
|
||||
info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true
|
||||
if (flags & 0x0080) {
|
||||
info.headersEncrypted = true
|
||||
break
|
||||
}
|
||||
} else if (type === R4_FILE) {
|
||||
if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break
|
||||
const large = Boolean(flags & 0x0100)
|
||||
let packed = data.readUInt32LE(p + 7)
|
||||
let unpacked = data.readUInt32LE(p + 11)
|
||||
const nameSize = data.readUInt16LE(p + 26)
|
||||
let nameStart = p + 32
|
||||
if (large) {
|
||||
packed += data.readUInt32LE(p + 32) * 2 ** 32
|
||||
unpacked += data.readUInt32LE(p + 36) * 2 ** 32
|
||||
nameStart = p + 40
|
||||
}
|
||||
const rawName = data.subarray(nameStart, nameStart + nameSize)
|
||||
info.entries.push({
|
||||
name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'),
|
||||
// 0xffffffff with LHD_LARGE unset means "unknown" in some archivers.
|
||||
size: unpacked === 0xffffffff && !large ? null : unpacked,
|
||||
packedSize: packed,
|
||||
directory: (flags & 0x00e0) === 0x00e0,
|
||||
encrypted: Boolean(flags & 0x0004),
|
||||
splitBefore: Boolean(flags & 0x0001),
|
||||
splitAfter: Boolean(flags & 0x0002),
|
||||
stored: data[p + 25] === 0x30,
|
||||
dataOffset: p + size,
|
||||
})
|
||||
next = p + size + packed
|
||||
}
|
||||
|
||||
if (next <= p) break
|
||||
p = next
|
||||
}
|
||||
// Archivers before RAR 3.0 don't set the first-volume flag; a first
|
||||
// volume is one whose first file does not continue from a previous one.
|
||||
if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
|
||||
info.firstVolume = true
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
/**
|
||||
* RAR 4 stores Unicode names as "ASCII\0<compressed UTF-16>", where the
|
||||
* compressed part refers back to the ASCII bytes (unrar's DecodeFileName).
|
||||
*/
|
||||
export function decodeRar4Name(raw: Buffer): string {
|
||||
const zero = raw.indexOf(0)
|
||||
if (zero === -1) return raw.toString('utf8')
|
||||
const ascii = raw.subarray(0, zero)
|
||||
const enc = raw.subarray(zero + 1)
|
||||
if (enc.length === 0) return ascii.toString('latin1')
|
||||
|
||||
const out: number[] = []
|
||||
const highByte = enc[0]
|
||||
let encPos = 1
|
||||
let flags = 0
|
||||
let flagBits = 0
|
||||
while (encPos < enc.length) {
|
||||
if (flagBits === 0) {
|
||||
flags = enc[encPos++]
|
||||
flagBits = 8
|
||||
}
|
||||
switch (flags >> 6) {
|
||||
case 0:
|
||||
out.push(enc[encPos++])
|
||||
break
|
||||
case 1:
|
||||
out.push(enc[encPos++] + (highByte << 8))
|
||||
break
|
||||
case 2:
|
||||
out.push(enc[encPos] + (enc[encPos + 1] << 8))
|
||||
encPos += 2
|
||||
break
|
||||
case 3: {
|
||||
let length = enc[encPos++]
|
||||
if (length & 0x80) {
|
||||
const correction = enc[encPos++]
|
||||
for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) {
|
||||
out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8))
|
||||
}
|
||||
} else {
|
||||
for (length += 2; length > 0 && out.length < ascii.length; length--) {
|
||||
out.push(ascii[out.length])
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
flags = (flags << 2) & 0xff
|
||||
flagBits -= 2
|
||||
}
|
||||
return String.fromCharCode(...out)
|
||||
}
|
||||
|
||||
// --- RAR 5.0 ----------------------------------------------------------------
|
||||
|
||||
const R5_MAIN = 1
|
||||
const R5_FILE = 2
|
||||
const R5_ENCRYPTION = 4
|
||||
const R5_END = 5
|
||||
|
||||
/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */
|
||||
function vint(data: Buffer, offset: number): { value: number; next: number } | null {
|
||||
let value = 0
|
||||
let shift = 1
|
||||
for (let i = offset; i < data.length && i < offset + 10; i++) {
|
||||
value += (data[i] & 0x7f) * shift
|
||||
shift *= 128
|
||||
if (!(data[i] & 0x80)) return { value, next: i + 1 }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function parseRar5(data: Buffer): RarInfo {
|
||||
const info: RarInfo = {
|
||||
format: 'rar5',
|
||||
volume: false,
|
||||
firstVolume: null,
|
||||
solid: false,
|
||||
headersEncrypted: false,
|
||||
entries: [],
|
||||
truncated: true,
|
||||
}
|
||||
let p = RAR5_MARKER.length
|
||||
|
||||
while (p + 5 <= data.length) {
|
||||
const headCrc = data.readUInt32LE(p)
|
||||
const headerSize = vint(data, p + 4)
|
||||
if (!headerSize || headerSize.value === 0) break
|
||||
const headerEnd = headerSize.next + headerSize.value
|
||||
if (headerEnd > data.length) break
|
||||
if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break
|
||||
|
||||
const type = vint(data, headerSize.next)
|
||||
const flags = type && vint(data, type.next)
|
||||
if (!type || !flags) break
|
||||
let cursor = flags.next
|
||||
let extraSize = 0
|
||||
let dataSize = 0
|
||||
if (flags.value & 0x0001) {
|
||||
const v = vint(data, cursor)
|
||||
if (!v) break
|
||||
extraSize = v.value
|
||||
cursor = v.next
|
||||
}
|
||||
if (flags.value & 0x0002) {
|
||||
const v = vint(data, cursor)
|
||||
if (!v) break
|
||||
dataSize = v.value
|
||||
cursor = v.next
|
||||
}
|
||||
|
||||
if (type.value === R5_END) {
|
||||
info.truncated = false
|
||||
break
|
||||
}
|
||||
if (type.value === R5_ENCRYPTION) {
|
||||
info.headersEncrypted = true
|
||||
break
|
||||
}
|
||||
|
||||
if (type.value === R5_MAIN) {
|
||||
const archiveFlags = vint(data, cursor)
|
||||
if (!archiveFlags) break
|
||||
info.volume = Boolean(archiveFlags.value & 0x0001)
|
||||
// A volume number is only stored for volumes after the first.
|
||||
info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true
|
||||
info.solid = Boolean(archiveFlags.value & 0x0004)
|
||||
} else if (type.value === R5_FILE) {
|
||||
const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize)
|
||||
if (!entry) break
|
||||
info.entries.push(entry)
|
||||
}
|
||||
|
||||
const next = headerEnd + dataSize
|
||||
if (next <= p) break
|
||||
p = next
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
function parseRar5File(
|
||||
data: Buffer,
|
||||
start: number,
|
||||
extraStart: number,
|
||||
headerEnd: number,
|
||||
headerFlags: number,
|
||||
dataSize: number
|
||||
): RarEntry | null {
|
||||
const fileFlags = vint(data, start)
|
||||
const unpacked = fileFlags && vint(data, fileFlags.next)
|
||||
const attributes = unpacked && vint(data, unpacked.next)
|
||||
if (!fileFlags || !unpacked || !attributes) return null
|
||||
let cursor = attributes.next
|
||||
if (fileFlags.value & 0x0002) cursor += 4 // mtime
|
||||
if (fileFlags.value & 0x0004) cursor += 4 // data CRC32
|
||||
const compression = vint(data, cursor)
|
||||
const hostOs = compression && vint(data, compression.next)
|
||||
const nameLength = hostOs && vint(data, hostOs.next)
|
||||
if (!compression || !hostOs || !nameLength) return null
|
||||
const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8')
|
||||
// Compression information: bits 7-9 are the method, 0 is "store".
|
||||
const method = Math.floor(compression.value / 128) & 0x7
|
||||
|
||||
// Extra area records: size, type, data. Type 1 is file encryption.
|
||||
let encrypted = false
|
||||
let e = extraStart
|
||||
while (e < headerEnd) {
|
||||
const recordSize = vint(data, e)
|
||||
const recordType = recordSize && vint(data, recordSize.next)
|
||||
if (!recordSize || !recordType || recordSize.value === 0) break
|
||||
if (recordType.value === 0x01) encrypted = true
|
||||
e = recordSize.next + recordSize.value
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
size: fileFlags.value & 0x0008 ? null : unpacked.value,
|
||||
packedSize: dataSize,
|
||||
directory: Boolean(fileFlags.value & 0x0001),
|
||||
encrypted,
|
||||
splitBefore: Boolean(headerFlags & 0x0008),
|
||||
splitAfter: Boolean(headerFlags & 0x0010),
|
||||
stored: method === 0,
|
||||
dataOffset: headerEnd,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The data of an entry stored without compression, copied out of the
|
||||
* archive buffer. Null when it is compressed, encrypted, split over volumes
|
||||
* or not completely in the buffer.
|
||||
*/
|
||||
export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null {
|
||||
if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
||||
if (entry.size === null || entry.size !== entry.packedSize) return null
|
||||
const end = entry.dataOffset + entry.packedSize
|
||||
if (end > data.length) return null
|
||||
return data.subarray(entry.dataOffset, end)
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
/**
|
||||
* Grouping of files into releases (downloads), independent of AdonisJS.
|
||||
*/
|
||||
export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'
|
||||
@@ -0,0 +1,211 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { looksLikeFilename } from '../subject/file_key.js'
|
||||
|
||||
export interface ReleaseIdentity {
|
||||
key: string
|
||||
/** Human-readable name: the subject text or the shared filename stem. */
|
||||
name: string
|
||||
/** Number of files announced by the `[i/M]` counter, when present. */
|
||||
fileTotal?: number
|
||||
/** Which rule recognised the release. */
|
||||
source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot'
|
||||
}
|
||||
|
||||
export interface ReleaseInput {
|
||||
poster: string
|
||||
/** Subject without segment counter (the binary's `subject`). */
|
||||
subject: string
|
||||
/** Filename of the binary. */
|
||||
name: string
|
||||
/** True when the identity came from the yEnc body (obfuscated post). */
|
||||
weak: boolean
|
||||
}
|
||||
|
||||
// Suffixes that differ between the files of one release. Stripped repeatedly,
|
||||
// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x".
|
||||
const SUFFIXES = [
|
||||
/\.vol\d+[+-]\d+\.par2$/i,
|
||||
/\.par2$/i,
|
||||
/\.part\d+\.rar$/i,
|
||||
/\.r\d{2,3}$/i,
|
||||
/\.rar$/i,
|
||||
/\.7z(\.\d{3})?$/i,
|
||||
/\.zip(\.\d{3})?$/i,
|
||||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i,
|
||||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||||
/\.tar$/i,
|
||||
/\.(zst|gz|xz|bz2)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||||
]
|
||||
|
||||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||||
export function fileStem(name: string): string {
|
||||
let stem = name.trim()
|
||||
let changed = true
|
||||
while (changed) {
|
||||
changed = false
|
||||
for (const suffix of SUFFIXES) {
|
||||
const next = stem.replace(suffix, '')
|
||||
if (next !== stem && next.length > 0) {
|
||||
stem = next
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return stem
|
||||
}
|
||||
|
||||
/**
|
||||
* What remains of a subject once the filename, counters, "yEnc" and sizes are
|
||||
* removed. Posters that name the release outside the quotes put it here.
|
||||
*/
|
||||
export function subjectRemainder(subject: string): string {
|
||||
return subject
|
||||
.replace(/"[^"]*"/g, ' ')
|
||||
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
|
||||
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
|
||||
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
|
||||
.replace(/\byEnc\b/gi, ' ')
|
||||
.replace(/[\s\-–.:|]+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Posting tools put random per-file tokens and poster IDs in subjects (e.g.
|
||||
* `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
|
||||
* name once those are removed identifies a release.
|
||||
*/
|
||||
function isRandomToken(token: string): boolean {
|
||||
const t = token.replace(/^\[|\]$/g, '')
|
||||
if (t.length < 6) return false
|
||||
if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true
|
||||
return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t)
|
||||
}
|
||||
|
||||
/**
|
||||
* A stable per-post token: one short run of letters/digits with at least one
|
||||
* digit. Requiring a digit keeps constant labels like `[PRiVATE]` from
|
||||
* grouping different posts of the same poster.
|
||||
*/
|
||||
export function isPostToken(remainder: string): boolean {
|
||||
const compact = remainder.replace(/[\s\[\]()_-]+/g, '')
|
||||
return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact)
|
||||
}
|
||||
|
||||
export function namesRelease(remainder: string): boolean {
|
||||
const words = remainder
|
||||
.split(/\s+/)
|
||||
.filter((w) => w && !isRandomToken(w))
|
||||
.join(' ')
|
||||
// At least two words of letters, joined the way release names are.
|
||||
return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words)
|
||||
}
|
||||
|
||||
function hash(...parts: (string | number | undefined)[]) {
|
||||
return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides which release (download) a file belongs to. Returns null when the
|
||||
* file cannot be grouped from its name alone (random extensionless names in
|
||||
* obfuscated posts; those need par2 matching). See CONCEPT.md (D16).
|
||||
*/
|
||||
export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
||||
const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
|
||||
const stem = fileStem(input.name)
|
||||
|
||||
if (input.weak) {
|
||||
// Obfuscating posters randomise the poster per article; a real filename
|
||||
// stem is specific enough on its own.
|
||||
if (!looksLikeFilename(input.name)) return null
|
||||
return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
|
||||
}
|
||||
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||||
const remainder = subjectRemainder(input.subject)
|
||||
if (namesRelease(remainder)) {
|
||||
return {
|
||||
key: hash('subject', input.poster, remainder),
|
||||
name: remainder,
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
}
|
||||
}
|
||||
// Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
|
||||
// [i/M] counter it groups the files of one post whose names are random.
|
||||
if (fileTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
|
||||
}
|
||||
// The [i/M] total is informational, not part of the key: posters often
|
||||
// upload the par2 index without a counter, and it belongs to the same post.
|
||||
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
|
||||
}
|
||||
|
||||
/** Release of a par2 recovery set; every file listed in it belongs to it. */
|
||||
export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity {
|
||||
return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' }
|
||||
}
|
||||
|
||||
const SEPARATOR = /[._ -]/
|
||||
|
||||
/**
|
||||
* A release name from the files a par2 set lists. Files that differ only in
|
||||
* the middle (episodes S03E01…S03E20, parts) give their shared name, with
|
||||
* the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to
|
||||
* SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato.
|
||||
* Otherwise the longest stem, which tends to be the most descriptive (the
|
||||
* video file).
|
||||
*/
|
||||
export function par2ReleaseName(fileNames: string[]): string | undefined {
|
||||
const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean)
|
||||
if (stems.length === 0) return undefined
|
||||
const longest = [...stems].sort((a, b) => b.length - a.length)[0]
|
||||
if (stems.length < 2) return longest
|
||||
|
||||
let prefix = stems.reduce((p, s) => {
|
||||
let i = 0
|
||||
while (i < p.length && i < s.length && p[i] === s[i]) i++
|
||||
return p.slice(0, i)
|
||||
})
|
||||
let suffix = stems.reduce((p, s) => {
|
||||
let i = 0
|
||||
while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++
|
||||
return p.slice(p.length - i)
|
||||
})
|
||||
const shortest = Math.min(...stems.map((s) => s.length))
|
||||
if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest)
|
||||
|
||||
// Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token.
|
||||
const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix)
|
||||
if (season) prefix = season[1]
|
||||
else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) {
|
||||
prefix = prefix.replace(/[^._ -]*$/, '')
|
||||
}
|
||||
suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '')
|
||||
prefix = prefix.replace(/[._ -]+$/, '')
|
||||
|
||||
const name = suffix ? prefix + suffix : prefix
|
||||
// Only when the shared part carries most of the name (not just "The.").
|
||||
return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest
|
||||
}
|
||||
|
||||
/**
|
||||
* Release of a RAR volume set, from what its headers list: the shared
|
||||
* top-level directory if there is one, otherwise the first entry. For
|
||||
* volumes after the first, the first entry is the file that continues from
|
||||
* the previous volume, so all volumes of a single-file archive agree.
|
||||
*/
|
||||
export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
|
||||
if (entries.length === 0) return null
|
||||
const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
const nested = entries.some((e) => /[\\/]/.test(e.name))
|
||||
if (nested && tops.size === 1) {
|
||||
const [top] = tops
|
||||
return { key: hash('rar-dir', top), name: top, source: 'rar' }
|
||||
}
|
||||
const first = entries[0]
|
||||
const base = first.name.split(/[\\/]/).pop() ?? first.name
|
||||
return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' }
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
/**
|
||||
* Separating equal-sized obfuscated volumes by posting order, independent of AdonisJS.
|
||||
*/
|
||||
export { splitInstances, splitByRank, splitVerified, verifyVolume, type SplitArticle, type SplitOptions } from './instances.js'
|
||||
@@ -0,0 +1,178 @@
|
||||
import { combineCrc32 } from '../yenc/decode.js'
|
||||
|
||||
export interface SplitArticle {
|
||||
number: number
|
||||
messageId: string
|
||||
bytes: number
|
||||
/** Article number in the group; posting order. */
|
||||
articleNumber: number | null
|
||||
/** yEnc part CRC and decoded part length, for verification. */
|
||||
pcrc32?: string | null
|
||||
partSize?: number | null
|
||||
/** Whole-file CRC32, usually only in the last part. */
|
||||
fileCrc32?: string | null
|
||||
}
|
||||
|
||||
export interface SplitOptions {
|
||||
/** Total parts per volume; a volume that has them all takes no more articles. */
|
||||
totalParts: number
|
||||
/**
|
||||
* Maximum distance in article numbers between consecutive articles of one
|
||||
* volume. Separates different posts that happen to share a size.
|
||||
*/
|
||||
window?: number
|
||||
/**
|
||||
* How far (in posting positions) an article may be from where its volume
|
||||
* predicts it; covers out-of-order finishing across connections.
|
||||
*/
|
||||
tolerance?: number
|
||||
}
|
||||
|
||||
interface Volume {
|
||||
articles: SplitArticle[]
|
||||
/** Posting rank of each part number present. */
|
||||
ranks: Map<number, number>
|
||||
lastArticle: number
|
||||
opened: number
|
||||
}
|
||||
|
||||
/** ~1 minute of posting in a busy group. */
|
||||
const DEFAULT_WINDOW = 20_000
|
||||
|
||||
/**
|
||||
* Separates the articles of several equal-sized files that share one
|
||||
* identity (obfuscated volumes with random names, possibly from different
|
||||
* posts) into one list per file. See CONCEPT.md (D20).
|
||||
*
|
||||
* Posting tools upload a queue (volume 1 parts 1..n, volume 2 parts 1..n,
|
||||
* ...) over several connections, so within a volume part k+1 lands about one
|
||||
* position after part k in posting order, give or take the connection count.
|
||||
* Each article, in posting order, goes to the eligible volume (lacks this
|
||||
* part, not complete, active within `window` article numbers) whose
|
||||
* predicted position for it is closest: taken from that volume's article with
|
||||
* the nearest part number, which keeps the prediction local and unaffected
|
||||
* by missing articles elsewhere. A deviation above the tolerance starts a new
|
||||
* volume. The result is recomputed from all articles, so it improves as late
|
||||
* articles arrive.
|
||||
*/
|
||||
export function splitInstances(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
|
||||
const window = options.window ?? DEFAULT_WINDOW
|
||||
const tolerance = options.tolerance ?? Math.max(16, Math.ceil(options.totalParts * 0.1))
|
||||
const seen = new Set<string>()
|
||||
const ordered = articles
|
||||
.filter((a) => (seen.has(a.messageId) ? false : (seen.add(a.messageId), true)))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
|
||||
a.messageId.localeCompare(b.messageId)
|
||||
)
|
||||
|
||||
const volumes: Volume[] = []
|
||||
ordered.forEach((article, rank) => {
|
||||
const position = article.articleNumber ?? Number.MAX_SAFE_INTEGER
|
||||
let best: Volume | undefined
|
||||
let bestScore: [number, number] | undefined
|
||||
for (const volume of volumes) {
|
||||
if (volume.ranks.has(article.number) || volume.ranks.size >= options.totalParts) continue
|
||||
if (position - volume.lastArticle > window) continue
|
||||
const deviation = Math.abs(rank - predictedRank(volume, article.number))
|
||||
if (deviation > tolerance) continue
|
||||
const score: [number, number] = [deviation, volume.opened]
|
||||
if (!bestScore || score[0] < bestScore[0] || (score[0] === bestScore[0] && score[1] < bestScore[1])) {
|
||||
best = volume
|
||||
bestScore = score
|
||||
}
|
||||
}
|
||||
if (!best) {
|
||||
best = { articles: [], ranks: new Map(), lastArticle: position, opened: volumes.length }
|
||||
volumes.push(best)
|
||||
}
|
||||
best.articles.push(article)
|
||||
best.ranks.set(article.number, rank)
|
||||
best.lastArticle = Math.max(best.lastArticle, position)
|
||||
})
|
||||
|
||||
return volumes.map((v) => v.articles.sort((a, b) => a.number - b.number))
|
||||
}
|
||||
|
||||
/** Where part `number` should be in posting order, judged from the volume's nearest part. */
|
||||
function predictedRank(volume: Volume, number: number): number {
|
||||
let anchorPart = 0
|
||||
let anchorRank = 0
|
||||
let nearest = Number.MAX_SAFE_INTEGER
|
||||
for (const [part, rank] of volume.ranks) {
|
||||
const distance = Math.abs(part - number)
|
||||
if (distance < nearest) {
|
||||
nearest = distance
|
||||
anchorPart = part
|
||||
anchorRank = rank
|
||||
}
|
||||
}
|
||||
return anchorRank + (number - anchorPart)
|
||||
}
|
||||
|
||||
/**
|
||||
* Simpler split: for every part number, the i-th article in posting order
|
||||
* belongs to volume i. Right for a single, complete post; used as a second
|
||||
* proposal for verification.
|
||||
*/
|
||||
export function splitByRank(articles: SplitArticle[]): SplitArticle[][] {
|
||||
const byNumber = new Map<number, SplitArticle[]>()
|
||||
const seen = new Set<string>()
|
||||
for (const article of articles) {
|
||||
if (seen.has(article.messageId)) continue
|
||||
seen.add(article.messageId)
|
||||
byNumber.set(article.number, [...(byNumber.get(article.number) ?? []), article])
|
||||
}
|
||||
const volumes: SplitArticle[][] = []
|
||||
for (const group of byNumber.values()) {
|
||||
group.sort(
|
||||
(a, b) =>
|
||||
(a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
|
||||
a.messageId.localeCompare(b.messageId)
|
||||
)
|
||||
group.forEach((article, i) => (volumes[i] ??= []).push(article))
|
||||
}
|
||||
return volumes.map((v) => v.sort((a, b) => a.number - b.number))
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a volume has every part and its combined part CRCs equal the
|
||||
* whole-file CRC32 its last part announces: then it is exactly one file.
|
||||
*/
|
||||
export function verifyVolume(volume: SplitArticle[], totalParts: number): boolean {
|
||||
if (volume.length !== totalParts) return false
|
||||
const sorted = [...volume].sort((a, b) => a.number - b.number)
|
||||
if (sorted.some((a, i) => a.number !== i + 1 || !a.pcrc32 || !a.partSize)) return false
|
||||
const expected = sorted[sorted.length - 1].fileCrc32
|
||||
if (!expected) return false
|
||||
const combined = combineCrc32(sorted.map((a) => ({ crc: a.pcrc32!, length: a.partSize! })))
|
||||
return combined === expected.toLowerCase().padStart(8, '0')
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits and keeps only volumes that verify (verifyVolume). Proposals from
|
||||
* both methods are tried; articles of verified volumes are taken out and the
|
||||
* rest is tried again, until nothing new verifies. Unverified articles are
|
||||
* left out: guessing wrong would produce a corrupt file.
|
||||
*/
|
||||
export function splitVerified(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
|
||||
const verified: SplitArticle[][] = []
|
||||
let remaining = articles
|
||||
for (let round = 0; round < 4 && remaining.length >= options.totalParts; round++) {
|
||||
const found: SplitArticle[][] = []
|
||||
const used = new Set<string>()
|
||||
for (const propose of [(a: SplitArticle[]) => splitInstances(a, options), splitByRank]) {
|
||||
for (const volume of propose(remaining.filter((a) => !used.has(a.messageId)))) {
|
||||
if (verifyVolume(volume, options.totalParts)) {
|
||||
found.push(volume)
|
||||
for (const article of volume) used.add(article.messageId)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found.length === 0) break
|
||||
verified.push(...found)
|
||||
remaining = remaining.filter((a) => !used.has(a.messageId))
|
||||
}
|
||||
return verified
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import { createHash, createPublicKey, verify } from 'node:crypto'
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* Spotnet spot headers (free.pt), after Spotweb's Services_Format_Parsing::parseHeader.
|
||||
*
|
||||
* From: Nick <[PUBKEY[.USERSIGNATURE]]@[CAT][KEYID][SUBCATS].[SIZE].[x].[DATE].[..].[SIGNATURE]>
|
||||
* Subject: Title | Tag
|
||||
*/
|
||||
export interface SpotHeader {
|
||||
messageId: string
|
||||
poster: string
|
||||
title: string
|
||||
tag: string
|
||||
/** 0 image/video, 1 sound, 2 games, 3 applications. */
|
||||
category: number
|
||||
keyId: number
|
||||
/** e.g. { a: [9], b: [3], c: [7, 10], d: [17], z: [0] } */
|
||||
subcats: Record<string, number[]>
|
||||
size: number
|
||||
/** Unix seconds. */
|
||||
stamp: number
|
||||
/** Everything after the @. */
|
||||
header: string
|
||||
headerSign: string
|
||||
selfSignedPubKey: string
|
||||
userSignature?: string
|
||||
}
|
||||
|
||||
/** Spotnet's base64 variant: `-s` for `/`, `-p` for `+`, padding left out. */
|
||||
export function spotUnprepareBase64(input: string): string {
|
||||
const padded = input.length % 4 ? input + '='.repeat(4 - (input.length % 4)) : input
|
||||
return padded.replace(/-s/g, '/').replace(/-p/g, '+')
|
||||
}
|
||||
|
||||
/** Parses the From/Subject of a spot. Null when it isn't one. */
|
||||
export function parseSpotHeader(from: string, subject: string, messageId: string, date?: string): SpotHeader | null {
|
||||
const lt = from.indexOf('<')
|
||||
if (lt === -1) return null
|
||||
const address = from.slice(lt + 1, from.endsWith('>') ? -1 : undefined)
|
||||
const at = address.lastIndexOf('@')
|
||||
if (at === -1) return null
|
||||
const header = address.slice(at + 1)
|
||||
const [pubKey, userSignature] = address.slice(0, at).split('.')
|
||||
|
||||
const fields = header.split('.')
|
||||
if (fields.length < 6) return null
|
||||
const category = Number.parseInt(fields[0][0], 10) - 1
|
||||
const keyId = Number.parseInt(fields[0][1], 10)
|
||||
if (!(category >= 0) || !(keyId >= 0)) return null
|
||||
|
||||
// a9b4c0d5 (FTD) or a09b04c00d05 (newer): a letter, then digits.
|
||||
const subcats: Record<string, number[]> = {}
|
||||
for (const [, letter, digits] of fields[0].slice(2).toLowerCase().matchAll(/([a-z])(\d+)/g)) {
|
||||
if ('abcdz'.includes(letter)) (subcats[letter] ??= []).push(Number.parseInt(digits, 10))
|
||||
}
|
||||
|
||||
// Newer keys: "Title | Tag". Key 1 (oldest): the tag is the last field.
|
||||
const parts = subject.split('|')
|
||||
let title: string
|
||||
let tag = ''
|
||||
if (keyId !== 1) {
|
||||
title = parts[0].trim()
|
||||
tag = parts.length > 1 ? parts[1].trim() : ''
|
||||
} else {
|
||||
tag = parts.length > 1 ? parts[parts.length - 1].trim() : subject.trim()
|
||||
title = parts.length > 1 ? parts.slice(0, -2).join('|').trim() : ''
|
||||
}
|
||||
|
||||
const parsedDate = date ? Date.parse(date) : Number.NaN
|
||||
return {
|
||||
messageId: messageId.replace(/^<|>$/g, ''),
|
||||
poster: from.slice(0, lt).trim(),
|
||||
title,
|
||||
tag,
|
||||
category,
|
||||
keyId,
|
||||
subcats,
|
||||
size: Number.parseInt(fields[1], 10) || 0,
|
||||
stamp: Number.isNaN(parsedDate) ? Number.parseInt(fields[3], 10) || 0 : Math.floor(parsedDate / 1000),
|
||||
header,
|
||||
headerSign: fields[fields.length - 1],
|
||||
selfSignedPubKey: spotUnprepareBase64(pubKey ?? ''),
|
||||
userSignature: userSignature ? spotUnprepareBase64(userSignature) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/** Spotnet's own signing keys (Spotweb settings `rsa_keys`), base64 modulus. */
|
||||
export const SPOTNET_KEYS: Record<number, string> = {
|
||||
2: 'ys8WSlqonQMWT8ubG0tAA2Q07P36E+CJmb875wSR1XH7IFhEi0CCwlUzNqBFhC+P',
|
||||
3: 'uiyChPV23eguLAJNttC/o0nAsxXgdjtvUvidV2JL+hjNzc4Tc/PPo2JdYvsqUsat',
|
||||
4: '1k6RNDVD6yBYWR6kHmwzmSud7JkNV4SMigBrs+jFgOK5Ldzwl17mKXJhl+su/GR9',
|
||||
}
|
||||
|
||||
const base64url = (b64: string) => Buffer.from(b64, 'base64').toString('base64url')
|
||||
|
||||
/** RSA PKCS#1 v1.5 with SHA-1, as Spotweb's checkRsaSignature (exponent AQAB). */
|
||||
export function checkRsaSignature(message: Buffer | string, signatureB64: string, modulusB64: string): boolean {
|
||||
try {
|
||||
const key = createPublicKey({ key: { kty: 'RSA', n: base64url(modulusB64), e: 'AQAB' }, format: 'jwk' })
|
||||
const data = typeof message === 'string' ? toBytes(message) : message
|
||||
return verify('sha1', data, key, Buffer.from(signatureB64, 'base64'))
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Spots are signed over their ISO-8859-1 bytes. */
|
||||
function toBytes(text: string): Buffer {
|
||||
return /^[\u0000-ÿ]*$/.test(text) ? Buffer.from(text, 'latin1') : Buffer.from(text, 'utf8')
|
||||
}
|
||||
|
||||
/** SPOTSIGN_V2 and self-signed messages carry a hashcash: sha1('<msgid>') starts with 0000. */
|
||||
function hashcash(messageId: string): boolean {
|
||||
return createHash('sha1').update(`<${messageId}>`).digest('hex').startsWith('0000')
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies a spot header like Spotweb does: key 7 (self-signed) needs a
|
||||
* hashcash and the poster's key, keys 2–4 Spotnet's keys; spots that also
|
||||
* carry a user signature must have it match the message id.
|
||||
*/
|
||||
export function verifySpotHeader(spot: SpotHeader, keys: Record<number, string> = SPOTNET_KEYS): boolean {
|
||||
if (!spot.title || !spot.poster) return false
|
||||
// Old key 1 spots from before 2011 weren't signed.
|
||||
if (spot.keyId === 1 && spot.stamp <= 1293870080) return true
|
||||
if (!spot.headerSign) return false
|
||||
|
||||
const signature = spotUnprepareBase64(spot.headerSign)
|
||||
const toCheck = spot.title + spot.header.slice(0, spot.header.length - spot.headerSign.length - 1) + spot.poster
|
||||
let verified: boolean
|
||||
if (spot.keyId === 7 || (spot.keyId === 2 && spot.selfSignedPubKey.length > 50)) {
|
||||
// Self-signed (and key-2 "dispose" messages): the hashcash is the gate, as in Spotweb.
|
||||
verified = hashcash(spot.messageId)
|
||||
} else {
|
||||
verified = keys[spot.keyId] !== undefined && checkRsaSignature(toCheck, signature, keys[spot.keyId])
|
||||
}
|
||||
if (verified && spot.userSignature && spot.selfSignedPubKey) {
|
||||
verified = checkRsaSignature(`<${spot.messageId}>`, spot.userSignature, spot.selfSignedPubKey)
|
||||
}
|
||||
return verified
|
||||
}
|
||||
|
||||
/** Spotweb's spotter id: base64 of the CRC32 (little-endian) of the poster's public key. */
|
||||
export function spotterId(pubKeyB64: string): string {
|
||||
if (!pubKeyB64) return ''
|
||||
const crc = Buffer.alloc(4)
|
||||
crc.writeUInt32LE(crc32(Buffer.from(pubKeyB64, 'base64')) >>> 0)
|
||||
return crc.toString('base64').replace(/[/+=]/g, '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Moderation: key-2 spots titled "delete|dispose|remove <message-id>".
|
||||
* From Spotnet's key they are moderator actions; with the poster's own key
|
||||
* (a spotter id) a personal dispose of their own spot.
|
||||
*/
|
||||
export function moderationTarget(spot: Pick<SpotHeader, 'keyId' | 'title'>): string | null {
|
||||
if (spot.keyId !== 2) return null
|
||||
const [command, target] = spot.title.split(' ')
|
||||
if (!['delete', 'dispose', 'remove'].includes(command?.toLowerCase()) || !target) return null
|
||||
return target.replace(/^<|>$/g, '')
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* Spotnet (free.pt) spots, independent of AdonisJS: header parsing and
|
||||
* signature checks after Spotweb, the spot XML and its compressed NZB.
|
||||
* See CONCEPT.md (D29).
|
||||
*/
|
||||
export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
|
||||
export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, type SpotXml } from './xml.js'
|
||||
@@ -0,0 +1,84 @@
|
||||
import { inflateRawSync } from 'node:zlib'
|
||||
|
||||
export interface SpotXml {
|
||||
title: string
|
||||
category: number | null
|
||||
subcats: string[]
|
||||
size: number
|
||||
website: string
|
||||
/** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
|
||||
nzbSegments: string[]
|
||||
created: number | null
|
||||
}
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
|
||||
|
||||
function text(value: string | undefined): string {
|
||||
if (value === undefined) return ''
|
||||
const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
|
||||
if (cdata) return cdata[1]
|
||||
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
if (entity[0] === '#') {
|
||||
const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
|
||||
}
|
||||
return ENTITIES[entity.toLowerCase()] ?? match
|
||||
})
|
||||
}
|
||||
|
||||
const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, 'i').exec(xml)?.[1]
|
||||
|
||||
/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */
|
||||
export function parseSpotXml(xml: string): SpotXml | null {
|
||||
const posting = element(xml, 'Posting')
|
||||
if (!posting) return null
|
||||
const categoryXml = element(posting, 'Category') ?? ''
|
||||
const key = Number.parseInt(element(posting, 'Key') ?? '', 10)
|
||||
const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
|
||||
const nzb = element(posting, 'NZB') ?? ''
|
||||
return {
|
||||
title: text(element(posting, 'Title')).trim(),
|
||||
// Newer keys number categories from 1.
|
||||
category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
|
||||
subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()),
|
||||
size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0,
|
||||
website: text(element(posting, 'Website')).trim(),
|
||||
nzbSegments: [...nzb.matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean),
|
||||
created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null,
|
||||
}
|
||||
}
|
||||
|
||||
/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */
|
||||
export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
|
||||
const headers: Record<string, string> = {}
|
||||
let xml = ''
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const colon = line.indexOf(':')
|
||||
if (colon <= 0) continue
|
||||
const name = line.slice(0, colon).toLowerCase()
|
||||
const value = line.slice(colon + 1).replace(/^ /, '')
|
||||
if (name === 'x-xml') xml += value
|
||||
else headers[name] ??= value
|
||||
}
|
||||
return { headers, xml }
|
||||
}
|
||||
|
||||
/**
|
||||
* Spot binaries (the NZB, images): article bodies joined without line
|
||||
* breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw
|
||||
* deflate (PHP gzinflate).
|
||||
*/
|
||||
export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer {
|
||||
const joined = bodies
|
||||
.map((body) =>
|
||||
body
|
||||
.toString('latin1')
|
||||
.split('\r\n')
|
||||
// NNTP dot-stuffing.
|
||||
.map((line) => (line.startsWith('..') ? line.slice(1) : line))
|
||||
.join('')
|
||||
)
|
||||
.join('')
|
||||
const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1')
|
||||
return compressed ? inflateRawSync(unescaped) : unescaped
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Identity of one uploaded file: every segment of it has the same poster, the
|
||||
* same subject apart from the segment counter, and the same segment count.
|
||||
* The group is deliberately left out so crossposts resolve to one file.
|
||||
* See CONCEPT.md (D12).
|
||||
*/
|
||||
export function fileKey(poster: string, base: string, totalSegments: number): string {
|
||||
return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex')
|
||||
}
|
||||
|
||||
/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */
|
||||
export function looksLikeFilename(name: string): boolean {
|
||||
return /\.[A-Za-z0-9]{1,5}$/.test(name) && !/^[0-9a-f]{32,}$/i.test(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of a file known only from its yEnc header (obfuscated posts).
|
||||
* The poster is left out: obfuscating posters randomise it per article.
|
||||
* - A real filename is stable across the articles of a file, and needed to
|
||||
* tell apart the equal-sized volumes of one archive.
|
||||
* - A random, extensionless name can change per article too; the exact
|
||||
* byte size and part count are what stays the same.
|
||||
* Collisions are caught by the conflict check in IngestService.
|
||||
*/
|
||||
export function weakFileKey(name: string, size: number, totalParts: number): string {
|
||||
const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
|
||||
return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Conventions of binary posts (subject lines, file identity), independent of AdonisJS.
|
||||
*/
|
||||
export { parseSubject, type ParsedSubject } from './parse.js'
|
||||
export { fileKey, weakFileKey, looksLikeFilename } from './file_key.js'
|
||||
@@ -0,0 +1,59 @@
|
||||
export interface ParsedSubject {
|
||||
/** Filename from the quoted part of the subject. */
|
||||
name: string
|
||||
/** Segment number `k` from `(k/N)`. */
|
||||
segment: number
|
||||
/** Segment count `N` from `(k/N)`. */
|
||||
totalSegments: number
|
||||
/** File counter `[i/M]`, when present. */
|
||||
fileIndex?: number
|
||||
fileTotal?: number
|
||||
/**
|
||||
* The subject with the segment counter removed. Identical for every segment
|
||||
* of one uploaded file, so it is part of the file identity (see fileKey).
|
||||
*/
|
||||
base: string
|
||||
}
|
||||
|
||||
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
|
||||
const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
|
||||
|
||||
/**
|
||||
* Parses a binary post subject such as
|
||||
* `[07/45] - "Some.Release.part06.rar" yEnc (12/137) 51315352`.
|
||||
*
|
||||
* Returns null for subjects without a quoted filename and segment counter
|
||||
* (obfuscated posts), which need their yEnc header read from the body instead.
|
||||
*/
|
||||
export function parseSubject(subject: string): ParsedSubject | null {
|
||||
const nameEnd = subject.lastIndexOf('"')
|
||||
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
if (!name) return null
|
||||
|
||||
// The segment counter is the last "(k/N)" after the filename; a file size may follow it.
|
||||
let counter: RegExpExecArray | undefined
|
||||
for (const match of subject.matchAll(SEGMENT_COUNTER)) {
|
||||
if (match.index > nameEnd) counter = match
|
||||
}
|
||||
if (!counter) return null
|
||||
|
||||
const segment = Number.parseInt(counter[1], 10)
|
||||
const totalSegments = Number.parseInt(counter[2], 10)
|
||||
if (totalSegments < 1 || segment < 1 || segment > totalSegments) return null
|
||||
|
||||
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
|
||||
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart))
|
||||
return {
|
||||
name,
|
||||
segment,
|
||||
totalSegments,
|
||||
fileIndex: fileCounter ? Number.parseInt(fileCounter[1], 10) : undefined,
|
||||
fileTotal: fileCounter ? Number.parseInt(fileCounter[2], 10) : undefined,
|
||||
base,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import yencode from 'yencode'
|
||||
import { YencCrcError, YencError } from './errors.js'
|
||||
import { parseYencMeta, type YencMeta } from './meta.js'
|
||||
|
||||
export interface DecodedSegment {
|
||||
meta: YencMeta
|
||||
/** Decoded bytes of this segment. */
|
||||
data: Buffer
|
||||
/** 0-based offset of `data` within the complete file. */
|
||||
offset: number
|
||||
}
|
||||
|
||||
export interface DecodeOptions {
|
||||
/**
|
||||
* Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
|
||||
* bodies, so this defaults to true.
|
||||
*/
|
||||
stripDots?: boolean
|
||||
/** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
|
||||
verifyCrc?: boolean
|
||||
/**
|
||||
* Accept a decoded length that differs from the declared size (the CRC is
|
||||
* still checked). Some posting tools declare a size that leaves out a
|
||||
* header, e.g. the 20-byte header of `nzbe` containers.
|
||||
*/
|
||||
lenientSize?: boolean
|
||||
}
|
||||
|
||||
export function crc32Hex(data: Buffer): string {
|
||||
return yencode.crc32(data).toString('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
|
||||
* without the data itself.
|
||||
*/
|
||||
export function combineCrc32(parts: { crc: string; length: number }[]): string {
|
||||
let combined: Buffer | undefined
|
||||
for (const part of parts) {
|
||||
const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
|
||||
combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
|
||||
}
|
||||
return (combined ?? Buffer.alloc(4)).toString('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes one yEnc article body and checks its size and CRC32 against the
|
||||
* `=yend` trailer.
|
||||
*/
|
||||
export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
|
||||
const { stripDots = true, verifyCrc = true, lenientSize = false } = options
|
||||
const meta = parseYencMeta(body)
|
||||
const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
|
||||
|
||||
if (lenientSize) {
|
||||
// Only the CRC below decides.
|
||||
} else if (data.length !== meta.end.size) {
|
||||
throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
|
||||
}
|
||||
if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
|
||||
throw new YencError(
|
||||
`Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
|
||||
)
|
||||
}
|
||||
|
||||
if (verifyCrc) {
|
||||
// Multi-part posts carry pcrc32 per part; single-part posts only crc32.
|
||||
const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
|
||||
if (expected) {
|
||||
const actual = crc32Hex(data)
|
||||
if (actual !== expected) throw new YencCrcError(expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export class YencError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'YencError'
|
||||
}
|
||||
}
|
||||
|
||||
export class YencCrcError extends YencError {
|
||||
constructor(
|
||||
public readonly expected: string,
|
||||
public readonly actual: string
|
||||
) {
|
||||
super(`CRC32 mismatch: expected ${expected}, got ${actual}`)
|
||||
this.name = 'YencCrcError'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* yEnc decoding, independent of AdonisJS. See CONCEPT.md (D4, D10).
|
||||
*/
|
||||
export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js'
|
||||
export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js'
|
||||
export { YencFile } from './yenc_file.js'
|
||||
export { YencError, YencCrcError } from './errors.js'
|
||||
@@ -0,0 +1,140 @@
|
||||
import { YencError } from './errors.js'
|
||||
|
||||
/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */
|
||||
export interface YencBegin {
|
||||
line: number
|
||||
size: number
|
||||
name: string
|
||||
part?: number
|
||||
total?: number
|
||||
}
|
||||
|
||||
/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */
|
||||
export interface YencPart {
|
||||
begin: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */
|
||||
export interface YencEnd {
|
||||
size: number
|
||||
part?: number
|
||||
pcrc32?: string
|
||||
crc32?: string
|
||||
}
|
||||
|
||||
export interface YencMeta {
|
||||
begin: YencBegin
|
||||
part?: YencPart
|
||||
end: YencEnd
|
||||
/** Byte range of the encoded data inside the article body. */
|
||||
dataStart: number
|
||||
dataEnd: number
|
||||
}
|
||||
|
||||
const CRLF = Buffer.from('\r\n')
|
||||
const BEGIN = Buffer.from('=ybegin ')
|
||||
const PART = Buffer.from('=ypart ')
|
||||
const END = Buffer.from('\r\n=yend ')
|
||||
|
||||
/**
|
||||
* Parses `key=value` pairs from a yEnc header line. `name` is always the last
|
||||
* field and runs to the end of the line, because filenames may contain spaces
|
||||
* and `=`.
|
||||
*/
|
||||
function parseFields(line: string): Record<string, string> {
|
||||
const fields: Record<string, string> = {}
|
||||
const nameIndex = line.search(/(^|\s)name=/)
|
||||
let rest = line
|
||||
if (nameIndex !== -1) {
|
||||
const nameStart = line.indexOf('name=', nameIndex) + 'name='.length
|
||||
fields.name = line.slice(nameStart).trim()
|
||||
rest = line.slice(0, nameIndex)
|
||||
}
|
||||
for (const match of rest.matchAll(/(\w+)=(\S+)/g)) {
|
||||
fields[match[1]] = match[2]
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
function toInt(fields: Record<string, string>, key: string, required: true): number
|
||||
function toInt(fields: Record<string, string>, key: string, required?: false): number | undefined
|
||||
function toInt(fields: Record<string, string>, key: string, required = false) {
|
||||
const raw = fields[key]
|
||||
if (raw === undefined) {
|
||||
if (required) throw new YencError(`Missing "${key}" in yEnc header`)
|
||||
return undefined
|
||||
}
|
||||
const value = Number.parseInt(raw, 10)
|
||||
if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function toCrc(fields: Record<string, string>, key: string) {
|
||||
const raw = fields[key]
|
||||
if (raw === undefined) return undefined
|
||||
if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`)
|
||||
return raw.toLowerCase().padStart(8, '0')
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of parts in the file. Many posters leave `total=` out of `=ybegin`;
|
||||
* then it follows from the part size, since all parts except the last one
|
||||
* are the same size.
|
||||
*/
|
||||
export function inferTotalParts(meta: Pick<YencMeta, 'begin' | 'part'>): number {
|
||||
const { begin, part } = meta
|
||||
if (begin.total) return begin.total
|
||||
if (!part || begin.part === undefined) return 1
|
||||
if (part.end >= begin.size) return begin.part
|
||||
return Math.ceil(begin.size / (part.end - part.begin + 1))
|
||||
}
|
||||
|
||||
function lineAt(buffer: Buffer, start: number) {
|
||||
const eol = buffer.indexOf(CRLF, start)
|
||||
const end = eol === -1 ? buffer.length : eol
|
||||
return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates and parses the yEnc header/trailer lines of one article body,
|
||||
* without decoding the data itself.
|
||||
*/
|
||||
export function parseYencMeta(body: Buffer): YencMeta {
|
||||
const beginIndex = body.indexOf(BEGIN)
|
||||
if (beginIndex === -1) throw new YencError('No =ybegin line found')
|
||||
|
||||
const beginLine = lineAt(body, beginIndex)
|
||||
const beginFields = parseFields(beginLine.text)
|
||||
const begin: YencBegin = {
|
||||
line: toInt(beginFields, 'line', true),
|
||||
size: toInt(beginFields, 'size', true),
|
||||
name: beginFields.name ?? '',
|
||||
part: toInt(beginFields, 'part'),
|
||||
total: toInt(beginFields, 'total'),
|
||||
}
|
||||
if (!begin.name) throw new YencError('Missing "name" in =ybegin line')
|
||||
|
||||
let dataStart = beginLine.next
|
||||
let part: YencPart | undefined
|
||||
if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) {
|
||||
const partLine = lineAt(body, dataStart)
|
||||
const partFields = parseFields(partLine.text)
|
||||
part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) }
|
||||
dataStart = partLine.next
|
||||
}
|
||||
|
||||
const endIndex = body.lastIndexOf(END)
|
||||
if (endIndex === -1 || endIndex < dataStart - CRLF.length) {
|
||||
throw new YencError('No =yend line found')
|
||||
}
|
||||
const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text)
|
||||
const end: YencEnd = {
|
||||
size: toInt(endFields, 'size', true),
|
||||
part: toInt(endFields, 'part'),
|
||||
pcrc32: toCrc(endFields, 'pcrc32'),
|
||||
crc32: toCrc(endFields, 'crc32'),
|
||||
}
|
||||
|
||||
return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) }
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { crc32Hex, type DecodedSegment } from './decode.js'
|
||||
import { YencCrcError, YencError } from './errors.js'
|
||||
import { inferTotalParts } from './meta.js'
|
||||
|
||||
/**
|
||||
* Assembles the decoded segments of one file into a single buffer. Segments
|
||||
* may be added in any order.
|
||||
*/
|
||||
export class YencFile {
|
||||
readonly name: string
|
||||
readonly size: number
|
||||
readonly totalParts: number
|
||||
|
||||
private readonly buffer: Buffer
|
||||
private readonly received = new Set<number>()
|
||||
private bytesWritten = 0
|
||||
private fileCrc?: string
|
||||
|
||||
constructor(name: string, size: number, totalParts = 1) {
|
||||
this.name = name
|
||||
this.size = size
|
||||
this.totalParts = totalParts
|
||||
this.buffer = Buffer.alloc(size)
|
||||
}
|
||||
|
||||
/** Creates a file sized from the metadata of any of its segments. */
|
||||
static fromSegment(segment: DecodedSegment): YencFile {
|
||||
const { begin } = segment.meta
|
||||
// Many posters leave total= out; it follows from the part size.
|
||||
return new YencFile(begin.name, begin.size, inferTotalParts(segment.meta))
|
||||
}
|
||||
|
||||
add(segment: DecodedSegment): void {
|
||||
const { meta, data, offset } = segment
|
||||
// Only the size is compared: obfuscating posters can give every segment
|
||||
// of one file a different random name.
|
||||
if (meta.begin.size !== this.size) {
|
||||
throw new YencError(
|
||||
`Segment of "${meta.begin.name}" (${meta.begin.size} bytes) does not belong to "${this.name}" (${this.size} bytes)`
|
||||
)
|
||||
}
|
||||
if (offset + data.length > this.size) {
|
||||
throw new YencError(`Segment at offset ${offset} overruns file size ${this.size}`)
|
||||
}
|
||||
|
||||
const partNumber = meta.begin.part ?? 1
|
||||
if (this.received.has(partNumber)) return
|
||||
|
||||
data.copy(this.buffer, offset)
|
||||
this.received.add(partNumber)
|
||||
this.bytesWritten += data.length
|
||||
if (meta.end.crc32) this.fileCrc = meta.end.crc32
|
||||
}
|
||||
|
||||
get partsReceived(): number {
|
||||
return this.received.size
|
||||
}
|
||||
|
||||
get isComplete(): boolean {
|
||||
return this.received.size === this.totalParts && this.bytesWritten === this.size
|
||||
}
|
||||
|
||||
/** Part numbers (1-based) that have not been added yet. */
|
||||
missingParts(): number[] {
|
||||
const missing: number[] = []
|
||||
for (let part = 1; part <= this.totalParts; part++) {
|
||||
if (!this.received.has(part)) missing.push(part)
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the assembled file. Throws if parts are missing or, when the poster
|
||||
* included a whole-file crc32, if it does not match.
|
||||
*/
|
||||
getBuffer(): Buffer {
|
||||
if (!this.isComplete) {
|
||||
throw new YencError(`File "${this.name}" is incomplete, missing parts: ${this.missingParts().join(', ')}`)
|
||||
}
|
||||
if (this.fileCrc) {
|
||||
const actual = crc32Hex(this.buffer)
|
||||
if (actual !== this.fileCrc) throw new YencCrcError(this.fileCrc, actual)
|
||||
}
|
||||
return this.buffer
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user