Rebuild: fresh AdonisJS 7 API app on Postgres
The rebuild branch starts over (REBUILD.md): the API starter kit, with Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a shared one in development) and the #lib alias. Phase 1's code is imported from the current app module by module. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
699395c51f
commit
96f0ade10e
264 files changed
+777
-24132
No files matched your search
@@ -1,86 +0,0 @@
|
||||
/**
|
||||
* Edge aggregation (CONCEPT.md D37): groups header segments into files in
|
||||
* memory, so a seeding node writes one row per file instead of one per
|
||||
* article. A file is done once every part number from 1 to its total has
|
||||
* an article; what never completes within the node's window (an upload cut
|
||||
* by the range edges, a random subject per article) leaves as fragments
|
||||
* for the regular ingest path, where it can meet its other parts.
|
||||
*/
|
||||
|
||||
/** What the aggregator needs of a segment (IngestSegment has all of it). */
|
||||
export interface AggregatableSegment {
|
||||
fileKey: string
|
||||
number: number
|
||||
totalParts: number
|
||||
messageId: string
|
||||
bytes: number
|
||||
}
|
||||
|
||||
export interface AggregatedFile<S extends AggregatableSegment> {
|
||||
fileKey: string
|
||||
/** One segment per part number, in part order; the first article seen wins (a later one is a repost). */
|
||||
segments: S[]
|
||||
}
|
||||
|
||||
interface OpenFile<S> {
|
||||
parts: Map<number, S>
|
||||
totalParts: number
|
||||
/** Batch in which a segment last arrived. */
|
||||
touched: number
|
||||
}
|
||||
|
||||
export class FileAggregator<S extends AggregatableSegment> {
|
||||
private open = new Map<string, OpenFile<S>>()
|
||||
private batch = 0
|
||||
/** Segments with a part number outside 1..total: never part of a complete file. */
|
||||
private stray: S[] = []
|
||||
|
||||
/** Adds one batch of segments; returns the files they completed. */
|
||||
add(segments: S[]): AggregatedFile<S>[] {
|
||||
this.batch++
|
||||
const touched = new Set<string>()
|
||||
for (const segment of segments) {
|
||||
if (segment.number < 1 || segment.number > segment.totalParts) {
|
||||
this.stray.push(segment)
|
||||
continue
|
||||
}
|
||||
let file = this.open.get(segment.fileKey)
|
||||
if (!file) {
|
||||
file = { parts: new Map(), totalParts: segment.totalParts, touched: this.batch }
|
||||
this.open.set(segment.fileKey, file)
|
||||
}
|
||||
if (!file.parts.has(segment.number)) file.parts.set(segment.number, segment)
|
||||
file.touched = this.batch
|
||||
touched.add(segment.fileKey)
|
||||
}
|
||||
|
||||
const complete: AggregatedFile<S>[] = []
|
||||
for (const key of touched) {
|
||||
const file = this.open.get(key)!
|
||||
if (file.parts.size < file.totalParts) continue
|
||||
this.open.delete(key)
|
||||
complete.push({ fileKey: key, segments: [...file.parts.values()].sort((a, b) => a.number - b.number) })
|
||||
}
|
||||
return complete
|
||||
}
|
||||
|
||||
/**
|
||||
* Gives up on files that got no segment in the last `idleBatches` batches
|
||||
* (all open files when 0) and returns their segments, plus stray ones.
|
||||
*/
|
||||
evict(idleBatches = 0): S[] {
|
||||
const out = this.stray
|
||||
this.stray = []
|
||||
for (const [key, file] of this.open) {
|
||||
if (idleBatches > 0 && this.batch - file.touched < idleBatches) continue
|
||||
this.open.delete(key)
|
||||
out.push(...file.parts.values())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Files still waiting for parts. */
|
||||
get openFiles(): number {
|
||||
return this.open.size
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
import { sevenZipVolume } from '../sevenzip/index.js'
|
||||
|
||||
/**
|
||||
* Files whose first segment says what an archive holds: named first RAR
|
||||
* volumes (.part01.rar, .rar), 7z (.7z, .7z.001) and `.bin`, an extension
|
||||
* uploaders use for archive volumes too (sniffed by content).
|
||||
*/
|
||||
export function isArchiveStart(name: string): boolean {
|
||||
if (/\.bin$/i.test(name)) return true
|
||||
const volume = sevenZipVolume(name)
|
||||
if (volume === 0 || volume === 1) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* A later RAR volume (.part02.rar, .r00) or a par2 recovery volume
|
||||
* (.vol03+04.par2): the volume number is readable even under a random stem.
|
||||
* The set is listed from its first volume and named from its index .par2,
|
||||
* so the first segment of these says nothing new (LOG.md 2026-10-04).
|
||||
*/
|
||||
export function isLaterVolume(name: string): boolean {
|
||||
const part = /\.part(\d+)\.rar$/i.exec(name)
|
||||
if (part) return Number(part[1]) > 1
|
||||
return /\.r\d{2,3}$/i.test(name) || /\.vol\d+[+-]\d+\.par2$/i.test(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* The set and 1-based number of a RAR volume: `x.part03.rar` is volume 3 of
|
||||
* `x`, and in the old naming `x.rar` is 1, `x.r00` 2, `x.r01` 3. Null for
|
||||
* other names.
|
||||
*/
|
||||
export function rarVolume(name: string): { base: string; number: number } | null {
|
||||
const part = /^(.*)\.part(\d+)\.rar$/i.exec(name)
|
||||
if (part) return { base: part[1].toLowerCase(), number: Number(part[2]) }
|
||||
const old = /^(.*)\.r(\d{2,3})$/i.exec(name)
|
||||
if (old) return { base: old[1].toLowerCase(), number: Number(old[2]) + 2 }
|
||||
const first = /^(.*)\.rar$/i.exec(name)
|
||||
if (first) return { base: first[1].toLowerCase(), number: 1 }
|
||||
return null
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
/**
|
||||
* Content recognition by magic bytes, independent of AdonisJS.
|
||||
*/
|
||||
export { sniffContentType, type ContentType } from './sniff.js'
|
||||
export { isArchiveStart, isLaterVolume, rarVolume } from './archive_names.js'
|
||||
export { parseSfv, sfvReleaseName, type SfvEntry } from './sfv.js'
|
||||
export { NICOVIDEO_FILE, NICOVIDEO_RELEASE, nicovideoInfo, nicovideoReleaseName, zstdPrefix, tarEntries, type NicovideoInfo } from './nicovideo.js'
|
||||
@@ -1,76 +0,0 @@
|
||||
import { createZstdDecompress } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* Captures of niconico videos (`<unixtime>_nicovideo_jp_watch_sm<id>.tar.zst`,
|
||||
* one poster, ~190k files on homelab). The tar's first entry after its
|
||||
* directory is `server_response.metadata`: the site's API response for the
|
||||
* watch page, 30–70 KB, so the first article of the file holds it.
|
||||
*/
|
||||
export const NICOVIDEO_FILE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)\.tar\.zst$/i
|
||||
export const NICOVIDEO_RELEASE = /^\d{9,11}_nicovideo_jp_watch_(sm\d+)$/i
|
||||
|
||||
export interface NicovideoInfo {
|
||||
id: string
|
||||
title: string
|
||||
owner: string | null
|
||||
registeredAt: string | null
|
||||
tags: string[]
|
||||
}
|
||||
|
||||
/** As much of a zstd stream as `data` (a prefix of it) decompresses to. */
|
||||
export function zstdPrefix(data: Buffer, maxBytes = 4 * 1024 * 1024): Promise<Buffer> {
|
||||
return new Promise((resolve) => {
|
||||
const out: Buffer[] = []
|
||||
let length = 0
|
||||
const z = createZstdDecompress()
|
||||
const done = () => resolve(Buffer.concat(out))
|
||||
z.on('data', (chunk: Buffer) => {
|
||||
out.push(chunk)
|
||||
length += chunk.length
|
||||
if (length >= maxBytes) z.destroy()
|
||||
})
|
||||
z.on('error', done)
|
||||
z.on('close', done)
|
||||
z.on('end', done)
|
||||
z.end(data)
|
||||
})
|
||||
}
|
||||
|
||||
/** Entries at the start of a (possibly truncated) ustar archive. */
|
||||
export function* tarEntries(tar: Buffer): Generator<{ name: string; data: Buffer }> {
|
||||
let offset = 0
|
||||
while (offset + 512 <= tar.length) {
|
||||
const header = tar.subarray(offset, offset + 512)
|
||||
const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '')
|
||||
if (!name) return
|
||||
const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, '')
|
||||
const size = Number.parseInt(header.subarray(124, 136).toString('latin1').replace(/\0.*$/s, '').trim() || '0', 8) || 0
|
||||
yield { name: prefix ? `${prefix}/${name}` : name, data: tar.subarray(offset + 512, offset + 512 + size) }
|
||||
offset += 512 + Math.ceil(size / 512) * 512
|
||||
}
|
||||
}
|
||||
|
||||
/** Title, uploader and tags from the start of a capture, or null when they aren't in it. */
|
||||
export function nicovideoInfo(tar: Buffer): NicovideoInfo | null {
|
||||
for (const entry of tarEntries(tar)) {
|
||||
if (!entry.name.endsWith('/server_response.metadata')) continue
|
||||
try {
|
||||
const text = entry.data.toString('utf8')
|
||||
const json = JSON.parse(text.slice(text.indexOf('{')))
|
||||
const video = json?.data?.response?.video
|
||||
if (!video?.title) return null
|
||||
const owner = json.data.response.owner?.nickname ?? json.data.response.channel?.name ?? json.data.metadata?.jsonLds?.[0]?.author?.name ?? null
|
||||
const tags = (json.data.response.tag?.items ?? []).map((t: any) => String(t.name)).filter(Boolean)
|
||||
return { id: String(video.id ?? ''), title: String(video.title).trim(), owner: owner ? String(owner).trim() : null, registeredAt: video.registeredAt ?? null, tags }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Release name for a capture: `<title> - <uploader> (nicovideo sm123)`. */
|
||||
export function nicovideoReleaseName(info: NicovideoInfo, fallbackId: string): string {
|
||||
const id = info.id || fallbackId
|
||||
return `${info.title}${info.owner ? ` - ${info.owner}` : ''} (nicovideo ${id})`
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { decodeSegment, YencCrcError, YencError, YencFile, type DecodedSegment } from '../yenc/index.js'
|
||||
import { isPar2, parsePar2, type Par2Set } from '../par2/index.js'
|
||||
import { parseRarHeaders, type RarEntry, type RarInfo } from '../rar/parse.js'
|
||||
import { extractEntry } from '../rar/extract.js'
|
||||
import { walkRar } from '../rar/walk.js'
|
||||
import { SegmentReader, type BodyFetcher } from '../rar/segment_reader.js'
|
||||
import { listSevenZip } from '../sevenzip/list.js'
|
||||
import { sniffContentType, type ContentType } from './sniff.js'
|
||||
import { NICOVIDEO_FILE, nicovideoInfo, zstdPrefix, type NicovideoInfo } from './nicovideo.js'
|
||||
|
||||
/**
|
||||
* Content inspection's download-and-parse half (CONCEPT.md D21, D41),
|
||||
* without a database: what the indexer needs to know about a file, from its
|
||||
* segments. The indexer prepares a task from the database, the probe runs
|
||||
* here or on a node (articles through the seed API's relay), and the
|
||||
* indexer applies the result. Tasks and results are plain JSON.
|
||||
*/
|
||||
|
||||
export interface TaskSegment {
|
||||
number: number
|
||||
id: string
|
||||
/** Article size in bytes. */
|
||||
size?: number
|
||||
}
|
||||
|
||||
/** Inspect one file: its first segment, and more where its kind needs it. */
|
||||
export interface InspectTask {
|
||||
kind: 'inspect'
|
||||
fileId: number
|
||||
/** real_name, else filename. */
|
||||
name: string
|
||||
segments: TaskSegment[]
|
||||
/** For a 7z listing: the archive's volume count and its last volume's segments; null when volumes are missing. */
|
||||
sevenZip: { volumes: number; last: TaskSegment[] } | null
|
||||
}
|
||||
|
||||
export interface InspectProbe {
|
||||
/** The whole file's size, from the yEnc header. */
|
||||
size: number
|
||||
contentType: ContentType | null
|
||||
md5_16k: string | null
|
||||
/** The file name in the yEnc header. */
|
||||
yencName: string
|
||||
nicovideo?: NicovideoInfo | null
|
||||
/** A whole small SFV (latin1) or NZB (utf8). */
|
||||
text?: string | null
|
||||
par2?: Par2Set | null
|
||||
archive?: RarInfo | null
|
||||
/** NZBs read from inside an unsplit archive, and how many NZB entries it lists. */
|
||||
archiveNzbs?: { entryName: string; xml: string }[]
|
||||
archiveNzbEntries?: number
|
||||
}
|
||||
|
||||
/** Walk a RAR set's headers for its complete listing (D40). */
|
||||
export interface WalkTask {
|
||||
kind: 'walk'
|
||||
releaseId: number
|
||||
volumes: TaskSegment[][]
|
||||
maxSegments: number
|
||||
}
|
||||
|
||||
export interface WalkProbe {
|
||||
entries: Omit<RarEntry, 'stored' | 'dataOffset'>[]
|
||||
complete: boolean
|
||||
stopped?: string
|
||||
headersEncrypted: boolean
|
||||
segments: number
|
||||
}
|
||||
|
||||
export type ContentTask = InspectTask | WalkTask
|
||||
|
||||
/** Small posted files (NZBs, SFVs) that are read completely. */
|
||||
const WHOLE_MAX_PARTS = 40
|
||||
const WHOLE_MAX_BYTES = 25 * 1024 * 1024
|
||||
/** Par2 index files are small; recovery volumes can be large and are not needed. */
|
||||
const PAR2_MAX_PARTS = 20
|
||||
const FIRST_CHUNK = 16 * 1024
|
||||
|
||||
/** MD5 of the first 16 KiB of a file, as par2 records it; undefined when the data is shorter than that but the file isn't. */
|
||||
export function chunkMd5(data: Buffer, fileSize: number): string | undefined {
|
||||
if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
|
||||
return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
|
||||
}
|
||||
|
||||
/** A "no such article" answer: 430 from the news server, 410 from the seed relay. */
|
||||
export function isMissingArticle(error: unknown): boolean {
|
||||
return /(^|[^0-9])(430|410)([^0-9]|$)/.test(String((error as any)?.message ?? error)) && !/timed out|no answer/i.test(String((error as any)?.message))
|
||||
}
|
||||
|
||||
/** Decodes a body; some tools declare a size that leaves out a header (nzbe), the CRC still has to match. */
|
||||
export function decodeBody(body: Buffer): DecodedSegment {
|
||||
try {
|
||||
return decodeSegment(body)
|
||||
} catch (error) {
|
||||
if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
|
||||
return decodeSegment(body, { lenientSize: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* All of a (small) file whose first segment is already decoded; null when it is too big.
|
||||
* The first segment isn't downloaded again (it was, for every par2 set, NZB and SFV).
|
||||
*/
|
||||
export async function readWhole(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Buffer | null> {
|
||||
const bytes = segments.reduce((sum, p) => sum + (p.size ?? 0), 0)
|
||||
if (segments.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
|
||||
if (segments.length === 1) return first.data
|
||||
const assembled = YencFile.fromSegment(first)
|
||||
assembled.add(first)
|
||||
for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id), { lenientSize: true }))
|
||||
return assembled.getBuffer()
|
||||
}
|
||||
|
||||
/** A par2 file whose first segment is already decoded, with its other parts if any; null when it is not par2. */
|
||||
export async function readPar2(first: DecodedSegment, segments: TaskSegment[], fetch: BodyFetcher): Promise<Par2Set | null> {
|
||||
if (!isPar2(first.data)) return null
|
||||
if (segments.length > PAR2_MAX_PARTS) return null
|
||||
if (segments.length === 1) return parsePar2(first.data)
|
||||
const assembled = YencFile.fromSegment(first)
|
||||
assembled.add(first)
|
||||
for (const part of segments.slice(1)) assembled.add(decodeSegment(await fetch(part.id)))
|
||||
return parsePar2(assembled.getBuffer())
|
||||
}
|
||||
|
||||
export function isNzbEntry(entry: Pick<RarEntry, 'name' | 'encrypted' | 'directory'>): boolean {
|
||||
return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
|
||||
}
|
||||
|
||||
/** NZBs that can be read from a single, unencrypted archive. */
|
||||
function hasNzbEntries(info: RarInfo): boolean {
|
||||
return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
|
||||
}
|
||||
|
||||
export async function probeInspection(
|
||||
task: InspectTask,
|
||||
fetch: BodyFetcher,
|
||||
warn: (message: string) => void = () => {}
|
||||
): Promise<InspectProbe> {
|
||||
let body: Buffer
|
||||
try {
|
||||
body = await fetch(task.segments[0].id)
|
||||
} catch (error: any) {
|
||||
// The server no longer has it (430; 410 through the seed relay), typically a takedown. A final
|
||||
// answer, and an expensive one: Eweka takes ~450 ms per missing article, so it isn't retried
|
||||
// (LOG.md 2026-10-06 (14)).
|
||||
if (isMissingArticle(error)) return { size: 0, contentType: 'missing', md5_16k: null, yencName: '' }
|
||||
throw error
|
||||
}
|
||||
const segment = decodeBody(body)
|
||||
const size = segment.meta.begin.size
|
||||
const contentType = sniffContentType(segment.data)
|
||||
const probe: InspectProbe = {
|
||||
size,
|
||||
contentType,
|
||||
md5_16k: chunkMd5(segment.data, size) ?? null,
|
||||
yencName: segment.meta.begin.name,
|
||||
}
|
||||
// An encrypted NZB, rclone crypt or yEncBin's container: nothing to read.
|
||||
if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') return probe
|
||||
if (NICOVIDEO_FILE.test(task.name)) {
|
||||
probe.nicovideo = nicovideoInfo(await zstdPrefix(segment.data))
|
||||
return probe
|
||||
}
|
||||
if (/\.sfv$/i.test(task.name)) {
|
||||
probe.text = (await readWhole(segment, task.segments, fetch))?.toString('latin1') ?? null
|
||||
return probe
|
||||
}
|
||||
if (contentType === 'nzb') {
|
||||
probe.text = (await readWhole(segment, task.segments, fetch))?.toString('utf8') ?? null
|
||||
return probe
|
||||
}
|
||||
probe.par2 = await readPar2(segment, task.segments, fetch)
|
||||
if (probe.par2) return probe
|
||||
if (contentType === '7z') {
|
||||
probe.archive = task.sevenZip ? await listSevenZip(segment.data, task.sevenZip.volumes, task.sevenZip.last, fetch, warn) : null
|
||||
return probe
|
||||
}
|
||||
const info = parseRarHeaders(segment.data)
|
||||
probe.archive = info
|
||||
if (info && hasNzbEntries(info)) {
|
||||
probe.archiveNzbs = []
|
||||
const archive = await readWhole(segment, task.segments, fetch)
|
||||
const archived = archive && parseRarHeaders(archive)
|
||||
const entries = archived ? archived.entries.filter(isNzbEntry) : []
|
||||
probe.archiveNzbEntries = entries.length
|
||||
for (const entry of entries) {
|
||||
const xml = await extractEntry(archive!, entry, undefined, warn)
|
||||
if (xml) probe.archiveNzbs.push({ entryName: entry.name, xml: xml.toString('utf8') })
|
||||
}
|
||||
}
|
||||
return probe
|
||||
}
|
||||
|
||||
export async function probeWalk(task: WalkTask, fetch: BodyFetcher): Promise<WalkProbe> {
|
||||
const reader = new SegmentReader(task.volumes, task.maxSegments, fetch)
|
||||
const walked = await walkRar(reader)
|
||||
return {
|
||||
entries: walked.entries.map(({ stored: _s, dataOffset: _d, ...entry }) => entry),
|
||||
complete: walked.complete,
|
||||
stopped: walked.stopped,
|
||||
headersEncrypted: walked.headersEncrypted,
|
||||
segments: reader.fetched,
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
/**
|
||||
* SFV checksum lists: one `name CRC32` per line, `;` comments. Scene
|
||||
* releases (music in particular) post one with every album; it names every
|
||||
* file that belongs to it.
|
||||
*/
|
||||
export interface SfvEntry {
|
||||
name: string
|
||||
crc: string
|
||||
}
|
||||
|
||||
export function parseSfv(text: string): SfvEntry[] {
|
||||
const entries: SfvEntry[] = []
|
||||
for (const raw of text.split(/\r?\n/)) {
|
||||
const line = raw.trim()
|
||||
if (!line || line.startsWith(';')) continue
|
||||
const match = /^(.+?)\s+([0-9a-f]{8})$/i.exec(line)
|
||||
if (match) entries.push({ name: match[1].replace(/^"|"$/g, '').split(/[\\/]/).pop()!, crc: match[2].toLowerCase() })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
/**
|
||||
* The album name an SFV gives a release: its stem without the leading
|
||||
* track number scene groups put on the playlist/SFV/cover files
|
||||
* (`00-artist-album-web-2026-grp.sfv` → `artist-album-web-2026-grp`).
|
||||
*/
|
||||
export function sfvReleaseName(sfvName: string): string {
|
||||
return sfvName.replace(/\.sfv$/i, '').replace(/^0{1,3}-/, '')
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
/**
|
||||
* Recognises a file type from its first bytes, independent of AdonisJS.
|
||||
* `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)):
|
||||
* "nzbe", 16 random bytes (salt/IV), then ciphertext.
|
||||
* `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com`
|
||||
* (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy.
|
||||
*/
|
||||
export type ContentType =
|
||||
/** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */
|
||||
| 'missing'
|
||||
| 'nzbe'
|
||||
| 'par2'
|
||||
| 'rar4'
|
||||
| 'rar5'
|
||||
| '7z'
|
||||
| 'zip'
|
||||
| 'zstd'
|
||||
| 'gzip'
|
||||
| 'xz'
|
||||
| 'matroska'
|
||||
| 'mp4'
|
||||
| 'rclone'
|
||||
| 'yencbin'
|
||||
| 'iso'
|
||||
| 'udf'
|
||||
| 'pdf'
|
||||
| 'nzb'
|
||||
| 'jpeg'
|
||||
| 'png'
|
||||
| 'mp3'
|
||||
| 'flac'
|
||||
| 'avi'
|
||||
| 'wav'
|
||||
| 'ogg'
|
||||
| 'bzip2'
|
||||
|
||||
/** Magic bytes at an offset; null matches any byte. */
|
||||
const SIGNATURES: [ContentType, number, (number | null)[]][] = [
|
||||
// Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…):
|
||||
// the bytes they share.
|
||||
['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]],
|
||||
['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]],
|
||||
['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]],
|
||||
['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]],
|
||||
['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]],
|
||||
['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]],
|
||||
['zip', 0, [0x50, 0x4b, 0x03, 0x04]],
|
||||
['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]],
|
||||
['gzip', 0, [0x1f, 0x8b]],
|
||||
['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]],
|
||||
['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]],
|
||||
['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box
|
||||
['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt
|
||||
['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]],
|
||||
['jpeg', 0, [0xff, 0xd8, 0xff]],
|
||||
['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]],
|
||||
['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag
|
||||
['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC"
|
||||
['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS"
|
||||
['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh"
|
||||
]
|
||||
|
||||
export function sniffContentType(data: Buffer): ContentType | null {
|
||||
for (const [type, offset, bytes] of SIGNATURES) {
|
||||
if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type
|
||||
}
|
||||
// Disc images, visible when the first part is big enough (sector 16 at 32 KiB):
|
||||
// ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03.
|
||||
if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso'
|
||||
if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' &&
|
||||
/^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf'
|
||||
// RIFF container: AVI or WAV by form type.
|
||||
if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') {
|
||||
const form = data.subarray(8, 12).toString('latin1')
|
||||
if (form === 'AVI ') return 'avi'
|
||||
if (form === 'WAVE') return 'wav'
|
||||
}
|
||||
const head = data.subarray(0, 512).toString('latin1')
|
||||
if (/^\s*(<\?xml[^>]*>\s*)?(<!DOCTYPE nzb|<nzb[\s>])/.test(head)) return 'nzb'
|
||||
return null
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Upload signatures (CONCEPT.md D42): which tool posted an article, read from
|
||||
* its Message-ID, and the group it was posted to first. Together they predict
|
||||
* whether a random-named upload will ever get a name: ngPost uploads in
|
||||
* a.b.ath almost never do, nyuu uploads in a.b.friends always do (LOG.md
|
||||
* 2026-10-08 (5)).
|
||||
*/
|
||||
const TOOLS: [string, RegExp][] = [
|
||||
// FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu: 12 random capital+small letter pairs, a ms timestamp
|
||||
['nyuu', /^(?:[A-Z][a-z]){12}-\d{13}@nyuu$/],
|
||||
['nyuu-host', /^(?:[A-Z][a-z]){12}-\d{13}@.+$/],
|
||||
// b94268bc702040f7b811d5318f727e4f@ngPost: a UUID's hex
|
||||
['ngPost', /^[0-9a-f]{32}@ngPost$/],
|
||||
['yuun', /^[0-9a-f]{32}@[0-9a-f]{32}\.yuun$/],
|
||||
['hex32', /^[0-9a-f]{32}@[A-Za-z0-9.-]+$/],
|
||||
['JBinUp', /@JBin(?:Up|Down)\.local$/],
|
||||
// part1of69.lOlWzvSHabJCBTVrs0ts@AdultPoster.local
|
||||
['partNofM', /^part\d+of\d+\./i],
|
||||
// 1790923856877$1729905757859$1@localhost
|
||||
['ts-dollar', /^\d{13}\$\d+\$\d+@localhost$/],
|
||||
// t3Y4mV92YCp0McxDc5@zLZlsw.RYqes: random local part and random host
|
||||
['random', /^[A-Za-z0-9]{10,40}@[A-Za-z0-9]{3,12}\.[A-Za-z0-9]{2,6}$/],
|
||||
]
|
||||
|
||||
/** The posting tool of a Message-ID (without angle brackets), or 'other'. */
|
||||
export function uploadTool(messageId: string | null | undefined): string {
|
||||
if (!messageId) return 'other'
|
||||
for (const [tool, pattern] of TOOLS) if (pattern.test(messageId)) return tool
|
||||
return 'other'
|
||||
}
|
||||
|
||||
/** The group of a signature: the first the upload was posted to, as stored in `groups`. */
|
||||
export function signatureGroup(groups: unknown): string {
|
||||
return Array.isArray(groups) && typeof groups[0] === 'string' ? groups[0] : ''
|
||||
}
|
||||
|
||||
/** The key of a signature in a Set. */
|
||||
export function signatureKey(tool: string, group: string) {
|
||||
return `${tool}\t${group}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an upload is in the share of dropped signatures that is still kept
|
||||
* (and probed), so their rates keep being measured. Decided per release key,
|
||||
* so a release is kept or dropped as a whole.
|
||||
*/
|
||||
export function explored(key: string, percent: number): boolean {
|
||||
if (percent <= 0) return false
|
||||
return createHash('md5').update(key).digest().readUInt32BE(0) % 10_000 < percent * 100
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
import { parseXrefGroups } from '../nntp/xref.js'
|
||||
import { fileKey, parseSubject, weakFileKey } from '../subject/index.js'
|
||||
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
|
||||
import { inferTotalParts, type YencMeta } from '../yenc/index.js'
|
||||
|
||||
/** One segment, with the identity of the file it belongs to already resolved. */
|
||||
export interface IngestSegment {
|
||||
fileKey: string
|
||||
poster: string
|
||||
/** Subject without segment counter. */
|
||||
subject: string
|
||||
name: string
|
||||
totalParts: number
|
||||
number: number
|
||||
/** Without angle brackets. */
|
||||
messageId: string
|
||||
bytes: number
|
||||
articleNumber?: bigint | number
|
||||
postedAt?: Date
|
||||
groups: string[]
|
||||
/**
|
||||
* The identity comes from the yEnc name only (obfuscated posts). Such names
|
||||
* can be shared by several equal-sized files of one post, so a second
|
||||
* article for the same part number marks the binary as conflicted.
|
||||
*/
|
||||
weakIdentity?: boolean
|
||||
/** Release (download) this file belongs to, when it can be determined. */
|
||||
release?: ReleaseIdentity | null
|
||||
/** Exact file size from =ybegin (body-derived segments only). */
|
||||
size?: number
|
||||
/** yEnc CRCs (body-derived segments only), to verify split volumes. */
|
||||
pcrc32?: string
|
||||
partSize?: number
|
||||
fileCrc32?: string
|
||||
/** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
|
||||
md5_16k?: string
|
||||
}
|
||||
|
||||
/** An XOVER overview entry as returned by nntp-js. */
|
||||
export type OverviewHeader = Record<string, string>
|
||||
|
||||
function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
|
||||
const xrefGroups = parseXrefGroups(header.xref)
|
||||
return {
|
||||
poster: header.from ?? '',
|
||||
messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
|
||||
bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
|
||||
articleNumber,
|
||||
postedAt: header.date ? new Date(header.date) : undefined,
|
||||
groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
||||
export function segmentFromHeader(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
articleNumber?: bigint | number
|
||||
): IngestSegment | null {
|
||||
const parsed = header.subject ? parseSubject(header.subject) : null
|
||||
if (!parsed || !header['message-id']) return null
|
||||
const base = common(header, groupName, articleNumber)
|
||||
return {
|
||||
...base,
|
||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||
subject: parsed.base,
|
||||
name: parsed.name,
|
||||
totalParts: parsed.totalSegments,
|
||||
number: parsed.segment,
|
||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a segment for an obfuscated post from the yEnc header in its body.
|
||||
* Subject and poster can be random per article, so the identity comes from
|
||||
* the yEnc header only (see weakFileKey).
|
||||
*/
|
||||
export function segmentFromYenc(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
|
||||
articleNumber?: bigint | number
|
||||
): IngestSegment | null {
|
||||
const base = common(header, groupName, articleNumber)
|
||||
const totalParts = inferTotalParts(meta)
|
||||
const number = meta.begin.part ?? 1
|
||||
if (!base.messageId || number < 1 || number > totalParts) return null
|
||||
const subject = `"${meta.begin.name}" yEnc`
|
||||
return {
|
||||
...base,
|
||||
fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
|
||||
subject,
|
||||
name: meta.begin.name,
|
||||
totalParts,
|
||||
number,
|
||||
weakIdentity: true,
|
||||
release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
|
||||
size: meta.begin.size,
|
||||
// Single-part posts only carry crc32, which is then also the part's CRC.
|
||||
pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
|
||||
partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
|
||||
fileCrc32: meta.end?.crc32,
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
/** Newznab categories this indexer assigns (see release_category() in SQL). */
|
||||
export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [
|
||||
{ id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] },
|
||||
{ id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] },
|
||||
{ id: 4000, name: 'PC', subcats: [] },
|
||||
{ id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] },
|
||||
{ id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] },
|
||||
{ id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] },
|
||||
]
|
||||
|
||||
/** Parent category of a (sub)category, e.g. 5040 → 5000. */
|
||||
export function parentCategory(id: number): number {
|
||||
return Math.floor(id / 1000) * 1000
|
||||
}
|
||||
|
||||
/** "TV > HD" style label. */
|
||||
export function categoryLabel(id: number): string {
|
||||
const parent = CATEGORIES.find((c) => c.id === parentCategory(id))
|
||||
if (!parent) return 'Other'
|
||||
const sub = parent.subcats.find((s) => s.id === id)
|
||||
return sub ? `${parent.name} > ${sub.name}` : parent.name
|
||||
}
|
||||
|
||||
/**
|
||||
* Expands requested categories (`cat=5000,2040`) to the concrete ones to
|
||||
* match: a parent includes all of its subcategories.
|
||||
*/
|
||||
export function expandCategories(requested: number[]): number[] {
|
||||
const result = new Set<number>()
|
||||
for (const id of requested) {
|
||||
const parent = CATEGORIES.find((c) => c.id === id)
|
||||
if (parent) {
|
||||
result.add(parent.id)
|
||||
for (const sub of parent.subcats) result.add(sub.id)
|
||||
} else {
|
||||
result.add(id)
|
||||
}
|
||||
}
|
||||
return [...result]
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
/**
|
||||
* Newznab API formatting, independent of AdonisJS. See CONCEPT.md (D23).
|
||||
*/
|
||||
export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js'
|
||||
export { capsXml, errorXml, resultsXml, type NewznabItem, type NewznabServer } from './xml.js'
|
||||
@@ -1,107 +0,0 @@
|
||||
import { create } from 'xmlbuilder2'
|
||||
import { CATEGORIES, categoryLabel, parentCategory } from './categories.js'
|
||||
|
||||
const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/'
|
||||
|
||||
export interface NewznabItem {
|
||||
id: number
|
||||
title: string
|
||||
category: number
|
||||
size: number
|
||||
files: number
|
||||
postedAt: Date | null
|
||||
poster: string | null
|
||||
groups: string[]
|
||||
password: boolean
|
||||
/** Longer text than the title, e.g. a Spotnet spot's description (UBB markup). */
|
||||
description?: string
|
||||
season?: string
|
||||
episode?: string
|
||||
}
|
||||
|
||||
export interface NewznabServer {
|
||||
title: string
|
||||
/** Absolute base URL, e.g. http://homelab:3333 */
|
||||
baseUrl: string
|
||||
/** Appended to download links so a client can fetch without extra config. */
|
||||
apiKey?: string
|
||||
}
|
||||
|
||||
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
|
||||
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
|
||||
export function ubbToText(ubb: string): string {
|
||||
return ubb
|
||||
.replace(/<br\s*\/?>|\[br\]/gi, '\n')
|
||||
.replace(/<\/?[a-z][^>]*>/gi, '')
|
||||
.replace(/\[url=([^\]]*)\]([\s\S]*?)\[\/url\]/gi, (_, url: string, label: string) => (label.trim() ? `${label} (${url.trim()})` : url.trim()))
|
||||
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]/gi, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
export function errorXml(code: number, description: string): string {
|
||||
return create({ version: '1.0', encoding: 'UTF-8' })
|
||||
.ele('error', { code: String(code), description })
|
||||
.end({ prettyPrint: true })
|
||||
}
|
||||
|
||||
export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string {
|
||||
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
|
||||
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
|
||||
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
|
||||
caps.ele('registration', { available: 'no', open: 'no' })
|
||||
const searching = caps.ele('searching')
|
||||
searching.ele('search', { available: 'yes', supportedParams: 'q' })
|
||||
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
|
||||
searching.ele('movie-search', { available: 'yes', supportedParams: 'q' })
|
||||
searching.ele('audio-search', { available: 'yes', supportedParams: 'q' })
|
||||
searching.ele('book-search', { available: 'yes', supportedParams: 'q' })
|
||||
const categories = caps.ele('categories')
|
||||
for (const category of CATEGORIES) {
|
||||
const element = categories.ele('category', { id: String(category.id), name: category.name })
|
||||
for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name })
|
||||
}
|
||||
return caps.end({ prettyPrint: true })
|
||||
}
|
||||
|
||||
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string {
|
||||
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
|
||||
'version': '2.0',
|
||||
'xmlns:atom': 'http://www.w3.org/2005/Atom',
|
||||
'xmlns:newznab': NEWZNAB_NS,
|
||||
})
|
||||
const channel = rss.ele('channel')
|
||||
channel.ele('title').txt(server.title)
|
||||
channel.ele('description').txt(`${server.title} API results`)
|
||||
channel.ele('link').txt(`${server.baseUrl}/`)
|
||||
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
|
||||
|
||||
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
|
||||
for (const item of items) {
|
||||
const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}`
|
||||
const pubDate = (item.postedAt ?? new Date()).toUTCString()
|
||||
const element = channel.ele('item')
|
||||
element.ele('title').txt(item.title)
|
||||
element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id))
|
||||
element.ele('link').txt(getUrl)
|
||||
element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`)
|
||||
element.ele('pubDate').txt(pubDate)
|
||||
element.ele('category').txt(categoryLabel(item.category))
|
||||
element.ele('description').txt(item.description ? ubbToText(item.description) : item.title)
|
||||
element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' })
|
||||
|
||||
const attr = (name: string, value: string | number) =>
|
||||
element.ele('newznab:attr', { name, value: String(value) })
|
||||
attr('category', parentCategory(item.category))
|
||||
if (item.category !== parentCategory(item.category)) attr('category', item.category)
|
||||
attr('size', item.size)
|
||||
attr('files', item.files)
|
||||
attr('grabs', 0)
|
||||
attr('password', item.password ? 1 : 0)
|
||||
if (item.poster) attr('poster', item.poster)
|
||||
if (item.groups.length) attr('group', item.groups[0])
|
||||
attr('usenetdate', pubDate)
|
||||
if (item.season) attr('season', item.season)
|
||||
if (item.episode) attr('episode', item.episode)
|
||||
}
|
||||
return rss.end({ prettyPrint: true })
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
/**
|
||||
* The article number where a group's posts reach `target` (ms since epoch),
|
||||
* by binary search between `first` and `last`. Article numbers only roughly
|
||||
* follow posting time, so the answer is within `precision` articles of the
|
||||
* first one posted at or after `target`.
|
||||
*
|
||||
* `dateOf(n)` returns the posting time of the first article at or after `n`
|
||||
* that still exists, or null when there is none nearby (expired or removed);
|
||||
* a gap counts as older than `target`.
|
||||
*/
|
||||
export async function articleAtDate(
|
||||
first: bigint,
|
||||
last: bigint,
|
||||
target: number,
|
||||
dateOf: (n: bigint) => Promise<number | null>,
|
||||
precision = 10_000n
|
||||
): Promise<bigint> {
|
||||
let lo = first
|
||||
let hi = last
|
||||
while (hi - lo > precision) {
|
||||
const mid = lo + (hi - lo) / 2n
|
||||
const date = await dateOf(mid)
|
||||
if (date === null || date < target) lo = mid
|
||||
else hi = mid
|
||||
}
|
||||
return lo
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
const CRLF = Buffer.from('\r\n')
|
||||
|
||||
/**
|
||||
* Splits a byte stream into messages that end in a delimiter, which the
|
||||
* consumer can change per message: `next(delimiter)` returns everything up
|
||||
* to the next `delimiter` (excluded). A replacement for nntp-js's
|
||||
* findByteSequence, which has two bugs:
|
||||
*
|
||||
* - An empty multi-line reply (`224 …\r\n.\r\n`, e.g. XOVER of a range
|
||||
* without articles) never ends: the status line takes the CRLF that
|
||||
* `\r\n.\r\n` starts with. Here, when a delimiter that starts with CRLF
|
||||
* follows one that ended with CRLF, that CRLF counts for both.
|
||||
* - A delimiter split over two socket chunks left its first bytes in the
|
||||
* returned data.
|
||||
*
|
||||
* Empty messages for a bare CRLF delimiter are skipped, as nntp-js does.
|
||||
*/
|
||||
export async function* delimitedReader(
|
||||
source: AsyncIterable<Uint8Array>,
|
||||
initial: Uint8Array
|
||||
): AsyncGenerator<Buffer, void, Uint8Array | undefined> {
|
||||
let delimiter = Buffer.from(initial)
|
||||
let parts: Buffer[] = []
|
||||
let length = 0
|
||||
// Bytes at the start of the buffer, already searched without a match.
|
||||
let searched = 0
|
||||
// Leading bytes that belong to the previous delimiter (a shared CRLF).
|
||||
let skip = 0
|
||||
|
||||
for await (const chunk of source) {
|
||||
parts.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength))
|
||||
length += chunk.byteLength
|
||||
|
||||
while (true) {
|
||||
const from = Math.max(0, searched - (delimiter.length - 1))
|
||||
const found = indexOf(parts, length, delimiter, from)
|
||||
if (found === -1) {
|
||||
searched = length
|
||||
break
|
||||
}
|
||||
const all = parts.length === 1 ? parts[0] : Buffer.concat(parts, length)
|
||||
const message = all.subarray(Math.min(skip, found), found)
|
||||
const rest = all.subarray(found + delimiter.length)
|
||||
const previous = delimiter
|
||||
|
||||
if (message.length === 0 && delimiter.equals(CRLF)) {
|
||||
// A stray empty line: keep looking for the same delimiter.
|
||||
} else {
|
||||
delimiter = Buffer.from((yield message) ?? delimiter)
|
||||
}
|
||||
|
||||
const shareCrlf = endsWithCrlf(previous) && delimiter.length > 2 && startsWithCrlf(delimiter)
|
||||
parts = shareCrlf ? [CRLF, rest] : [rest]
|
||||
length = rest.length + (shareCrlf ? 2 : 0)
|
||||
skip = shareCrlf ? 2 : 0
|
||||
searched = 0
|
||||
}
|
||||
}
|
||||
|
||||
if (length > skip) yield Buffer.concat(parts, length).subarray(skip)
|
||||
}
|
||||
|
||||
const endsWithCrlf = (b: Buffer) => b.length >= 2 && b[b.length - 2] === 13 && b[b.length - 1] === 10
|
||||
const startsWithCrlf = (b: Buffer) => b.length >= 2 && b[0] === 13 && b[1] === 10
|
||||
|
||||
/** indexOf over a list of buffers (their total `length`), starting at `from`; copies only the searched tail. */
|
||||
function indexOf(parts: Buffer[], length: number, needle: Buffer, from: number): number {
|
||||
if (parts.length === 1) return parts[0].indexOf(needle, from)
|
||||
// Collect the parts that cover [from, length).
|
||||
let start = length
|
||||
let i = parts.length
|
||||
while (i > 0 && start > from) start -= parts[--i].length
|
||||
const window = Buffer.concat(parts.slice(i), length - start)
|
||||
const found = window.indexOf(needle, from - start)
|
||||
return found === -1 ? -1 : start + found
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
export interface GroupInfo {
|
||||
count: bigint
|
||||
first: bigint
|
||||
last: bigint
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a `211 count first last name` GROUP reply (RFC 3977 §6.1.1).
|
||||
*
|
||||
* nntp-js's own `group()` destructures this reply without skipping the status
|
||||
* code, so its `first`/`last` are off by one field. Pass it `result.resp`.
|
||||
*/
|
||||
export function parseGroupResponse(resp: string): GroupInfo {
|
||||
const match = /^211\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)/.exec(resp.trim())
|
||||
if (!match) throw new Error(`Unexpected GROUP response: ${resp}`)
|
||||
return {
|
||||
count: BigInt(match[1]),
|
||||
first: BigInt(match[2]),
|
||||
last: BigInt(match[3]),
|
||||
name: match[4],
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import { NNTP } from 'nntp-js'
|
||||
import { delimitedReader } from './delimited_reader.js'
|
||||
|
||||
const CRLF = new Uint8Array([13, 10])
|
||||
|
||||
/**
|
||||
* nntp-js with its reply reader replaced by `delimitedReader` (see there for
|
||||
* the bugs it fixes; the empty-reply one hung XOVER of empty ranges forever).
|
||||
* The reader is created in `connect`, so that is overridden: the same steps
|
||||
* as nntp-js 1.0.4, with the other reader.
|
||||
*/
|
||||
const proto = NNTP.prototype as any
|
||||
if (!proto.__delimitedReader) {
|
||||
proto.__delimitedReader = true
|
||||
proto.connect = async function (this: any, readermode = false, timeout: number | undefined = undefined, socket?: any) {
|
||||
try {
|
||||
this.sock = socket ?? this._createSocket(timeout)
|
||||
// The first reply is always the one-line welcome.
|
||||
this.byteReader = delimitedReader(this.sock, CRLF)
|
||||
this.welcome = new TextDecoder('utf-8').decode(await this._getresp())
|
||||
this.caps = undefined
|
||||
this.connected = true
|
||||
await this.getcapabilities()
|
||||
this.readermodeAfterauth = false
|
||||
if (readermode && !this.caps.READER) {
|
||||
await this._setreadermode()
|
||||
if (!this.readermodeAfterauth) {
|
||||
this.caps = undefined
|
||||
await this.getcapabilities()
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.sock?.end()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export { NNTP }
|
||||
@@ -1,238 +0,0 @@
|
||||
import { promisify } from 'node:util'
|
||||
import net from 'node:net'
|
||||
import tls from 'node:tls'
|
||||
import { deflate, deflateSync, unzipSync } from 'node:zlib'
|
||||
|
||||
const deflateAsync = promisify(deflate)
|
||||
|
||||
export interface XoverClientOptions {
|
||||
host: string
|
||||
port: number
|
||||
user?: string
|
||||
password?: string
|
||||
secure?: boolean
|
||||
/** Hostname the TLS certificate must match, when connecting through a tunnel (default: host). */
|
||||
servername?: string
|
||||
/** A reply with no data for this long fails, and the connection with it (inactivity, not duration). */
|
||||
timeoutMs?: number
|
||||
/** Ask for gzip-compressed overview data (XFEATURE COMPRESS GZIP), when the server has it. */
|
||||
compress?: boolean
|
||||
}
|
||||
|
||||
export type Overview = [number, Record<string, string>]
|
||||
|
||||
/** The overview fields in RFC 3977 order, named as nntp-js names them (segmentFromHeader reads these). */
|
||||
const FIELDS = ['subject', 'from', 'date', 'message-id', 'references', ':bytes', ':lines', 'xref']
|
||||
const TERMINATOR = Buffer.from('\r\n.\r\n')
|
||||
const DOT_LINE = Buffer.from('.\r\n')
|
||||
|
||||
/**
|
||||
* A minimal NNTP client for seeding nodes (CONCEPT.md D37): one connection,
|
||||
* GROUP and XOVER only, with gzip-compressed overviews. nntp-js can't read
|
||||
* those: after "224 … [COMPRESS=GZIP]" the lines come as one zlib stream,
|
||||
* followed by a plain "." line; an empty range is a 420 without body.
|
||||
* About six times less traffic per header (LOG.md 2026-10-05).
|
||||
*/
|
||||
export class XoverClient {
|
||||
private chunks: Buffer[] = []
|
||||
private length = 0
|
||||
private wake?: () => void
|
||||
private failure?: Error
|
||||
compressed = false
|
||||
|
||||
private constructor(
|
||||
private sock: net.Socket,
|
||||
private timeoutMs: number
|
||||
) {
|
||||
sock.on('data', (chunk: Buffer) => {
|
||||
this.chunks.push(chunk)
|
||||
this.length += chunk.length
|
||||
this.wake?.()
|
||||
})
|
||||
const fail = (error: Error) => {
|
||||
this.failure ??= error
|
||||
this.wake?.()
|
||||
}
|
||||
sock.on('error', fail)
|
||||
sock.on('close', () => fail(new Error('NNTP connection closed')))
|
||||
sock.on('timeout', () => {
|
||||
fail(new Error('NNTP reply timed out'))
|
||||
sock.destroy()
|
||||
})
|
||||
}
|
||||
|
||||
static async connect(options: XoverClientOptions): Promise<XoverClient> {
|
||||
const sock = options.secure
|
||||
? tls.connect({ host: options.host, port: options.port, servername: options.servername || options.host })
|
||||
: net.connect({ host: options.host, port: options.port })
|
||||
const client = new XoverClient(sock, options.timeoutMs ?? 120_000)
|
||||
sock.setTimeout(client.timeoutMs)
|
||||
try {
|
||||
await client.expect(await client.line(), ['200', '201'])
|
||||
if (options.user) {
|
||||
const reply = await client.command(`AUTHINFO USER ${options.user}`)
|
||||
if (reply.startsWith('381')) await client.expect(await client.command(`AUTHINFO PASS ${options.password ?? ''}`), ['281'])
|
||||
else await client.expect(reply, ['281'])
|
||||
}
|
||||
if (options.compress) client.compressed = (await client.command('XFEATURE COMPRESS GZIP')).startsWith('290')
|
||||
} catch (error) {
|
||||
client.close()
|
||||
throw error
|
||||
}
|
||||
// Idle between commands is fine (a node waiting for the seed API); only replies are timed.
|
||||
sock.setTimeout(0)
|
||||
return client
|
||||
}
|
||||
|
||||
async group(name: string): Promise<{ count: number; low: number; high: number }> {
|
||||
return this.timed(() => this.selectGroup(name))
|
||||
}
|
||||
|
||||
/** Overviews of articles `from`..`to` in the current group; an empty range gives []. */
|
||||
async xover(from: number, to: number): Promise<Overview[]> {
|
||||
return this.timed(() => this.overviews(from, to))
|
||||
}
|
||||
|
||||
/**
|
||||
* Overviews of `from`..`to` as one zlib/gzip stream, as the server sent it when it
|
||||
* compresses (plain replies are deflated here): the seed API passes these to nodes
|
||||
* without unpacking them (CONCEPT.md D38). `parseOverviews(unzipSync(raw).toString())`.
|
||||
*/
|
||||
async xoverRaw(from: number, to: number): Promise<Buffer> {
|
||||
return this.timed(async () => {
|
||||
const reply = await this.command(`XOVER ${from}-${to}`)
|
||||
if (reply.startsWith('420') || reply.startsWith('423')) return deflateSync(Buffer.alloc(0))
|
||||
await this.expect(reply, ['224'])
|
||||
if (reply.includes('COMPRESS=GZIP')) return (await this.compressedRaw()).raw
|
||||
// Plain replies (the relay's default): compressed off the event loop, at the fastest level.
|
||||
return deflateAsync(Buffer.from(await this.plainBody()), { level: 1 })
|
||||
})
|
||||
}
|
||||
|
||||
/** A reply that stalls for timeoutMs fails the connection; idle time between commands doesn't count. */
|
||||
private async timed<T>(exchange: () => Promise<T>): Promise<T> {
|
||||
this.sock.setTimeout(this.timeoutMs)
|
||||
try {
|
||||
return await exchange()
|
||||
} finally {
|
||||
this.sock.setTimeout(0)
|
||||
}
|
||||
}
|
||||
|
||||
private async selectGroup(name: string): Promise<{ count: number; low: number; high: number }> {
|
||||
const reply = await this.command(`GROUP ${name}`)
|
||||
await this.expect(reply, ['211'])
|
||||
const [, count, low, high] = reply.split(' ')
|
||||
return { count: Number(count), low: Number(low), high: Number(high) }
|
||||
}
|
||||
|
||||
private async overviews(from: number, to: number): Promise<Overview[]> {
|
||||
const reply = await this.command(`XOVER ${from}-${to}`)
|
||||
if (reply.startsWith('420') || reply.startsWith('423')) return []
|
||||
await this.expect(reply, ['224'])
|
||||
const data = reply.includes('COMPRESS=GZIP') ? await this.compressedBody() : await this.plainBody()
|
||||
return parseOverviews(data)
|
||||
}
|
||||
|
||||
close() {
|
||||
try {
|
||||
this.sock.end('QUIT\r\n')
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
this.sock.destroy()
|
||||
}
|
||||
|
||||
private async command(line: string): Promise<string> {
|
||||
this.sock.write(`${line}\r\n`)
|
||||
return this.line()
|
||||
}
|
||||
|
||||
private async expect(reply: string, codes: string[]) {
|
||||
if (!codes.some((code) => reply.startsWith(code))) throw new Error(`NNTP: unexpected reply "${reply}"`)
|
||||
}
|
||||
|
||||
private buffer(): Buffer {
|
||||
if (this.chunks.length > 1) this.chunks = [Buffer.concat(this.chunks, this.length)]
|
||||
return this.chunks[0] ?? Buffer.alloc(0)
|
||||
}
|
||||
|
||||
private take(n: number): Buffer {
|
||||
const all = this.buffer()
|
||||
const out = all.subarray(0, n)
|
||||
const rest = all.subarray(n)
|
||||
this.chunks = rest.length > 0 ? [rest] : []
|
||||
this.length = rest.length
|
||||
return out
|
||||
}
|
||||
|
||||
private async more() {
|
||||
if (this.failure) throw this.failure
|
||||
await new Promise<void>((resolve) => (this.wake = resolve))
|
||||
this.wake = undefined
|
||||
if (this.failure && this.length === 0) throw this.failure
|
||||
}
|
||||
|
||||
private async line(): Promise<string> {
|
||||
while (true) {
|
||||
const at = this.buffer().indexOf('\r\n')
|
||||
if (at !== -1) return this.take(at + 2).subarray(0, at).toString('latin1')
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
|
||||
/** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */
|
||||
private async plainBody(): Promise<string> {
|
||||
let searched = 0
|
||||
while (true) {
|
||||
const all = this.buffer()
|
||||
// An empty body is just ".\r\n".
|
||||
if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) {
|
||||
this.take(3)
|
||||
return ''
|
||||
}
|
||||
const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
|
||||
if (at !== -1) return this.take(at + TERMINATOR.length).subarray(0, at + 2).toString('utf8').replace(/^\.\./gm, '.')
|
||||
searched = all.length
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
|
||||
private async compressedBody(): Promise<string> {
|
||||
return (await this.compressedRaw()).text
|
||||
}
|
||||
|
||||
/** A zlib/gzip stream followed by a "." line: complete once it ends in ".\r\n" and inflates. */
|
||||
private async compressedRaw(): Promise<{ raw: Buffer; text: string }> {
|
||||
while (true) {
|
||||
const all = this.buffer()
|
||||
if (all.length > DOT_LINE.length && all.subarray(all.length - DOT_LINE.length).equals(DOT_LINE)) {
|
||||
const raw = all.subarray(0, all.length - DOT_LINE.length)
|
||||
try {
|
||||
const text = unzipSync(raw).toString('utf8')
|
||||
this.take(all.length)
|
||||
return { raw: Buffer.from(raw), text }
|
||||
} catch {
|
||||
// ".\r\n" inside the compressed data: wait for more.
|
||||
}
|
||||
}
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Overview lines (tab-separated, after the article number) as nntp-js returns them. */
|
||||
export function parseOverviews(data: string): Overview[] {
|
||||
const out: Overview[] = []
|
||||
for (const line of data.split('\r\n')) {
|
||||
if (!line) continue
|
||||
const [number, ...tokens] = line.split('\t')
|
||||
const fields: Record<string, string> = {}
|
||||
for (let i = 0; i < FIELDS.length && i < tokens.length; i++) {
|
||||
// Full fields after the defaults carry their name: "Xref: host group:number".
|
||||
fields[FIELDS[i]] = i >= 7 ? tokens[i].replace(/^[^:\s]+:\s*/, '') : tokens[i]
|
||||
}
|
||||
out.push([Number.parseInt(number, 10), fields])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
/**
|
||||
* Returns the group names from an Xref header,
|
||||
* e.g. `news.eweka.nl alt.binaries.test:123 alt.binaries.x:456`.
|
||||
*/
|
||||
export function parseXrefGroups(xref: string | undefined): string[] {
|
||||
if (!xref) return []
|
||||
return xref
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.slice(1)
|
||||
.map((entry) => entry.slice(0, entry.lastIndexOf(':')))
|
||||
.filter(Boolean)
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
import { create } from 'xmlbuilder2'
|
||||
import type { Segment } from './segments.js'
|
||||
|
||||
export interface NzbFile {
|
||||
/** Name the downloader should use; the real name when it is known. */
|
||||
name: string
|
||||
poster: string
|
||||
/** Posting time in milliseconds. */
|
||||
date: number
|
||||
groups: string[]
|
||||
segments: Segment[]
|
||||
}
|
||||
|
||||
export interface NzbOptions {
|
||||
/** Shown by downloaders as the job name. */
|
||||
title?: string
|
||||
/** Archive password, when known. */
|
||||
password?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an NZB 1.1 document (newzbin DTD). The subject follows the common
|
||||
* `"name" yEnc (1/N)` form, which downloaders use to name the file.
|
||||
*/
|
||||
export function buildNzb(files: NzbFile[], options: NzbOptions = {}): string {
|
||||
const root = create({ version: '1.0', encoding: 'UTF-8' })
|
||||
.dtd({ pubID: '-//newzBin//DTD NZB 1.1//EN', sysID: 'http://www.newzbin.com/DTD/nzb/nzb-1.1.dtd' })
|
||||
.ele('nzb', { xmlns: 'http://www.newzbin.com/DTD/2003/nzb' })
|
||||
|
||||
if (options.title || options.password) {
|
||||
const head = root.ele('head')
|
||||
if (options.title) head.ele('meta', { type: 'name' }).txt(options.title)
|
||||
if (options.password) head.ele('meta', { type: 'password' }).txt(options.password)
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const sorted = [...file.segments].sort((a, b) => a.number - b.number)
|
||||
const element = root.ele('file', {
|
||||
poster: file.poster,
|
||||
date: String(Math.floor(file.date / 1000)),
|
||||
subject: `"${file.name}" yEnc (1/${sorted.length})`,
|
||||
})
|
||||
const groups = element.ele('groups')
|
||||
for (const group of file.groups) groups.ele('group').txt(group)
|
||||
const segments = element.ele('segments')
|
||||
for (const segment of sorted) {
|
||||
segments.ele('segment', { bytes: String(segment.size), number: String(segment.number) }).txt(segment.id)
|
||||
}
|
||||
}
|
||||
return root.end({ prettyPrint: true })
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
/**
|
||||
* NZB generation, independent of AdonisJS.
|
||||
*/
|
||||
export { buildNzb, type NzbFile, type NzbOptions } from './build.js'
|
||||
export { parseNzb, type ParsedNzb, type ParsedNzbFile } from './parse.js'
|
||||
export type { Segment } from './segments.js'
|
||||
@@ -1,72 +0,0 @@
|
||||
export interface ParsedNzbFile {
|
||||
poster: string
|
||||
/** Posting time in seconds (as in the NZB). */
|
||||
date: number
|
||||
subject: string
|
||||
groups: string[]
|
||||
segments: { number: number; bytes: number; id: string }[]
|
||||
}
|
||||
|
||||
export interface ParsedNzb {
|
||||
/** <head><meta type="..."> values, e.g. name, password, category. */
|
||||
meta: Record<string, string>
|
||||
files: ParsedNzbFile[]
|
||||
}
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
|
||||
|
||||
function decode(text: string): string {
|
||||
return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
if (entity[0] === '#') {
|
||||
const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
|
||||
}
|
||||
return ENTITIES[entity.toLowerCase()] ?? match
|
||||
})
|
||||
}
|
||||
|
||||
function attributes(tag: string): Record<string, string> {
|
||||
const result: Record<string, string> = {}
|
||||
for (const m of tag.matchAll(/([\w:-]+)\s*=\s*("([^"]*)"|'([^']*)')/g)) result[m[1]] = decode(m[3] ?? m[4] ?? '')
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses an NZB (1.0/1.1 and the older 0.9 DTD). A tolerant scanner rather
|
||||
* than a full XML parser: NZBs are flat, and this never resolves DTDs or
|
||||
* external entities. Returns null when there is no <nzb> root.
|
||||
*/
|
||||
export function parseNzb(xml: string, limits = { maxFiles: 20_000 }): ParsedNzb | null {
|
||||
if (!/<nzb[\s>]/i.test(xml)) return null
|
||||
|
||||
const meta: Record<string, string> = {}
|
||||
const head = /<head>([\s\S]*?)<\/head>/i.exec(xml)?.[1] ?? ''
|
||||
for (const m of head.matchAll(/<meta\s([^>]*)>([\s\S]*?)<\/meta>/gi)) {
|
||||
const type = attributes(m[1]).type
|
||||
if (type) meta[type.toLowerCase()] = decode(m[2].trim())
|
||||
}
|
||||
|
||||
const files: ParsedNzbFile[] = []
|
||||
for (const m of xml.matchAll(/<file\s([^>]*)>([\s\S]*?)<\/file>/gi)) {
|
||||
if (files.length >= limits.maxFiles) break
|
||||
const attrs = attributes(m[1])
|
||||
const body = m[2]
|
||||
const groups = [...body.matchAll(/<group>([^<]*)<\/group>/gi)].map((g) => decode(g[1].trim())).filter(Boolean)
|
||||
const segments = [...body.matchAll(/<segment\s([^>]*)>([^<]*)<\/segment>/gi)]
|
||||
.map((s) => {
|
||||
const a = attributes(s[1])
|
||||
return { number: Number(a.number), bytes: Number(a.bytes) || 0, id: decode(s[2].trim()).replace(/^<|>$/g, '') }
|
||||
})
|
||||
.filter((s) => Number.isInteger(s.number) && s.number > 0 && s.id)
|
||||
.sort((x, y) => x.number - y.number)
|
||||
if (segments.length === 0) continue
|
||||
files.push({
|
||||
poster: attrs.poster ?? '',
|
||||
date: Number(attrs.date) || 0,
|
||||
subject: attrs.subject ?? '',
|
||||
groups,
|
||||
segments,
|
||||
})
|
||||
}
|
||||
return { meta, files }
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
import { zstdCompressSync, zstdDecompressSync, constants } from 'node:zlib'
|
||||
|
||||
/** One article of a posted file: its number, message-id (without <>) and size in bytes. */
|
||||
export interface Segment {
|
||||
number: number
|
||||
id: string
|
||||
size: number
|
||||
}
|
||||
|
||||
const VERSION = 1
|
||||
|
||||
/**
|
||||
* Segment lists as stored in `files.segments` (CONCEPT.md D31): a version
|
||||
* byte, then zstd (level 1) of a JSON line with the sizes (and the numbers,
|
||||
* unless they are 1..N) followed by one message-id per line. About 20 bytes
|
||||
* per segment, against ~38 for the pglz-compressed JSON object it replaces:
|
||||
* message-ids are mostly random, so this is close to what's possible.
|
||||
*/
|
||||
export function packSegments(segments: Segment[]): Buffer {
|
||||
const sorted = [...segments].sort((a, b) => a.number - b.number)
|
||||
const sequential = sorted.every((s, i) => s.number === i + 1)
|
||||
const head = JSON.stringify(sequential ? { s: sorted.map((s) => s.size) } : { s: sorted.map((s) => s.size), n: sorted.map((s) => s.number) })
|
||||
const text = Buffer.from([head, ...sorted.map((s) => s.id)].join('\n'), 'utf8')
|
||||
const packed = zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 1 } })
|
||||
return Buffer.concat([Buffer.from([VERSION]), packed])
|
||||
}
|
||||
|
||||
export function unpackSegments(data: Buffer): Segment[] {
|
||||
if (data[0] !== VERSION) throw new Error(`Unknown segment list version ${data[0]}`)
|
||||
const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
|
||||
const { s: sizes, n: numbers } = JSON.parse(head) as { s: number[]; n?: number[] }
|
||||
return sizes.map((size, i) => ({ number: numbers?.[i] ?? i + 1, id: ids[i], size }))
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
/**
|
||||
* PAR2 parsing, independent of AdonisJS. Used to recover the real filenames
|
||||
* and sizes of obfuscated posts. See CONCEPT.md (D16).
|
||||
*/
|
||||
export { parsePar2, isPar2, type Par2File, type Par2Set } from './parse.js'
|
||||
@@ -1,74 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
export interface Par2File {
|
||||
/** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */
|
||||
fileId: string
|
||||
/** MD5 of the whole file (hex). */
|
||||
md5: string
|
||||
/** MD5 of the first 16 KiB (hex). */
|
||||
md5_16k: string
|
||||
length: number
|
||||
name: string
|
||||
}
|
||||
|
||||
export interface Par2Set {
|
||||
/** Recovery set ID (hex), shared by every packet of one par2 set. */
|
||||
recoverySetId: string
|
||||
files: Par2File[]
|
||||
}
|
||||
|
||||
const MAGIC = Buffer.from('PAR2\0PKT', 'latin1')
|
||||
const HEADER_SIZE = 64
|
||||
const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1')
|
||||
|
||||
export function isPar2(data: Buffer): boolean {
|
||||
return data.subarray(0, MAGIC.length).equals(MAGIC)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the FileDesc packets of a par2 file (index or recovery volume).
|
||||
* Packets with a bad length or checksum are skipped; parsing resumes at the
|
||||
* next magic sequence. Returns null when no valid FileDesc packet is found.
|
||||
*/
|
||||
export function parsePar2(data: Buffer): Par2Set | null {
|
||||
const files = new Map<string, Par2File>()
|
||||
let recoverySetId: string | undefined
|
||||
let offset = data.indexOf(MAGIC)
|
||||
|
||||
while (offset !== -1 && offset + HEADER_SIZE <= data.length) {
|
||||
const length = Number(data.readBigUInt64LE(offset + 8))
|
||||
const valid =
|
||||
length >= HEADER_SIZE &&
|
||||
length % 4 === 0 &&
|
||||
offset + length <= data.length &&
|
||||
createHash('md5')
|
||||
.update(data.subarray(offset + 32, offset + length))
|
||||
.digest()
|
||||
.equals(data.subarray(offset + 16, offset + 32))
|
||||
|
||||
if (!valid) {
|
||||
offset = data.indexOf(MAGIC, offset + 1)
|
||||
continue
|
||||
}
|
||||
|
||||
const setId = data.subarray(offset + 32, offset + 48).toString('hex')
|
||||
const type = data.subarray(offset + 48, offset + 64)
|
||||
if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) {
|
||||
recoverySetId = setId
|
||||
const body = data.subarray(offset + HEADER_SIZE, offset + length)
|
||||
const fileId = body.subarray(0, 16).toString('hex')
|
||||
files.set(fileId, {
|
||||
fileId,
|
||||
md5: body.subarray(16, 32).toString('hex'),
|
||||
md5_16k: body.subarray(32, 48).toString('hex'),
|
||||
length: Number(body.readBigUInt64LE(48)),
|
||||
// The name is padded with NULs to a multiple of 4 bytes.
|
||||
name: body.subarray(56).toString('utf8').replace(/\0+$/, ''),
|
||||
})
|
||||
}
|
||||
offset += length
|
||||
}
|
||||
|
||||
if (!recoverySetId || files.size === 0) return null
|
||||
return { recoverySetId, files: [...files.values()] }
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import { storedEntryData, type RarEntry } from './parse.js'
|
||||
|
||||
const run = promisify(execFile)
|
||||
|
||||
/** Tools that print one archive entry to stdout, tried in order. */
|
||||
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
|
||||
['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
|
||||
['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
|
||||
]
|
||||
|
||||
/**
|
||||
* The unpacked data of one entry of a complete (single-volume) archive:
|
||||
* copied out when stored, else unpacked with bsdtar (libarchive, in the
|
||||
* Docker image) or unrar. Null when that is not possible.
|
||||
*/
|
||||
export async function extractEntry(
|
||||
archive: Buffer,
|
||||
entry: RarEntry,
|
||||
maxBytes = 64 * 1024 * 1024,
|
||||
warn: (message: string) => void = () => {}
|
||||
): Promise<Buffer | null> {
|
||||
const stored = storedEntryData(archive, entry)
|
||||
if (stored) return stored
|
||||
if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
||||
if (entry.size !== null && entry.size > maxBytes) return null
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
|
||||
try {
|
||||
const file = path.join(dir, 'archive.rar')
|
||||
await writeFile(file, archive)
|
||||
for (const [tool, args] of EXTRACTORS) {
|
||||
try {
|
||||
const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
|
||||
if (stdout.length > 0) return stdout
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') warn(`${tool} could not extract "${entry.name}": ${error.message}`)
|
||||
}
|
||||
}
|
||||
return null
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
/**
|
||||
* RAR (4.x and 5.0) header parsing, independent of AdonisJS. Lists archive
|
||||
* contents from the first segment of a first volume. See CONCEPT.md (D19).
|
||||
*/
|
||||
export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js'
|
||||
export { walkRar, type VolumeReader, type RarWalk } from './walk.js'
|
||||
export { SegmentReader, type BodyFetcher } from './segment_reader.js'
|
||||
export { extractEntry } from './extract.js'
|
||||
@@ -1,351 +0,0 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
export interface RarEntry {
|
||||
name: string
|
||||
/** Unpacked size in bytes (null when the archiver did not record it). */
|
||||
size: number | null
|
||||
packedSize: number
|
||||
directory: boolean
|
||||
/** The file's data is password protected. */
|
||||
encrypted: boolean
|
||||
/** Continues from the previous volume. */
|
||||
splitBefore: boolean
|
||||
/** Continues in the next volume. */
|
||||
splitAfter: boolean
|
||||
/** Stored without compression, so the data can be copied out as is. */
|
||||
stored: boolean
|
||||
/** Offset of the entry's (packed) data in the parsed buffer. */
|
||||
dataOffset: number
|
||||
}
|
||||
|
||||
export interface RarInfo {
|
||||
/** 7z listings take the same shape (see lib/sevenzip). */
|
||||
format: 'rar4' | 'rar5' | '7z'
|
||||
/** Part of a multi-volume set. */
|
||||
volume: boolean
|
||||
/** First volume of the set (null when the format version does not say). */
|
||||
firstVolume: boolean | null
|
||||
solid: boolean
|
||||
/** File names are encrypted too; nothing can be listed without the password. */
|
||||
headersEncrypted: boolean
|
||||
entries: RarEntry[]
|
||||
/** The buffer ended before the end-of-archive header (normal for one segment). */
|
||||
truncated: boolean
|
||||
/**
|
||||
* Offset of the first block that was not read: where the next header
|
||||
* starts (possibly beyond the buffer, after an entry's data), or the
|
||||
* header the buffer cut off. After the end-of-archive header, its end.
|
||||
*/
|
||||
next: number
|
||||
}
|
||||
|
||||
const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00])
|
||||
const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00])
|
||||
|
||||
export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null {
|
||||
if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5'
|
||||
if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4'
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists a RAR archive from its headers only, so the first segment of a first
|
||||
* volume is enough. Returns null when the data is not a RAR archive.
|
||||
*
|
||||
* With `continueAs`, `data` starts at a header in the middle of a volume (no
|
||||
* signature), e.g. where a previous parse said the next header is.
|
||||
*/
|
||||
export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null {
|
||||
if (continueAs === 'rar4') return parseRar4(data, 0)
|
||||
if (continueAs === 'rar5') return parseRar5(data, 0)
|
||||
const format = rarFormat(data)
|
||||
if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length)
|
||||
if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length)
|
||||
return null
|
||||
}
|
||||
|
||||
// --- RAR 4.x (RAR 1.5 – 4.x archive format) -------------------------------
|
||||
|
||||
const R4_MAIN = 0x73
|
||||
const R4_FILE = 0x74
|
||||
const R4_END = 0x7b
|
||||
const R4_LONG_BLOCK = 0x8000
|
||||
|
||||
function parseRar4(data: Buffer, start: number): RarInfo {
|
||||
const info: RarInfo = {
|
||||
format: 'rar4',
|
||||
volume: false,
|
||||
firstVolume: null,
|
||||
solid: false,
|
||||
headersEncrypted: false,
|
||||
entries: [],
|
||||
truncated: true,
|
||||
next: start,
|
||||
}
|
||||
let p = start
|
||||
|
||||
while (p + 7 <= data.length) {
|
||||
const headCrc = data.readUInt16LE(p)
|
||||
const type = data[p + 2]
|
||||
const flags = data.readUInt16LE(p + 3)
|
||||
const size = data.readUInt16LE(p + 5)
|
||||
if (size < 7 || p + size > data.length) break
|
||||
|
||||
if (type === R4_END) {
|
||||
info.truncated = false
|
||||
p += size
|
||||
break
|
||||
}
|
||||
|
||||
let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0)
|
||||
|
||||
if (type === R4_MAIN) {
|
||||
info.volume = Boolean(flags & 0x0001)
|
||||
info.solid = Boolean(flags & 0x0008)
|
||||
info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true
|
||||
if (flags & 0x0080) {
|
||||
info.headersEncrypted = true
|
||||
break
|
||||
}
|
||||
} else if (type === R4_FILE) {
|
||||
if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break
|
||||
const large = Boolean(flags & 0x0100)
|
||||
let packed = data.readUInt32LE(p + 7)
|
||||
let unpacked = data.readUInt32LE(p + 11)
|
||||
const nameSize = data.readUInt16LE(p + 26)
|
||||
let nameStart = p + 32
|
||||
if (large) {
|
||||
packed += data.readUInt32LE(p + 32) * 2 ** 32
|
||||
unpacked += data.readUInt32LE(p + 36) * 2 ** 32
|
||||
nameStart = p + 40
|
||||
}
|
||||
const rawName = data.subarray(nameStart, nameStart + nameSize)
|
||||
info.entries.push({
|
||||
name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'),
|
||||
// 0xffffffff with LHD_LARGE unset means "unknown" in some archivers.
|
||||
size: unpacked === 0xffffffff && !large ? null : unpacked,
|
||||
packedSize: packed,
|
||||
directory: (flags & 0x00e0) === 0x00e0,
|
||||
encrypted: Boolean(flags & 0x0004),
|
||||
splitBefore: Boolean(flags & 0x0001),
|
||||
splitAfter: Boolean(flags & 0x0002),
|
||||
stored: data[p + 25] === 0x30,
|
||||
dataOffset: p + size,
|
||||
})
|
||||
next = p + size + packed
|
||||
}
|
||||
|
||||
if (next <= p) break
|
||||
p = next
|
||||
}
|
||||
// Archivers before RAR 3.0 don't set the first-volume flag; a first
|
||||
// volume is one whose first file does not continue from a previous one.
|
||||
if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) {
|
||||
info.firstVolume = true
|
||||
}
|
||||
info.next = p
|
||||
return info
|
||||
}
|
||||
|
||||
/**
|
||||
* RAR 4 stores Unicode names as "ASCII\0<compressed UTF-16>", where the
|
||||
* compressed part refers back to the ASCII bytes (unrar's DecodeFileName).
|
||||
*/
|
||||
export function decodeRar4Name(raw: Buffer): string {
|
||||
const zero = raw.indexOf(0)
|
||||
if (zero === -1) return raw.toString('utf8')
|
||||
const ascii = raw.subarray(0, zero)
|
||||
const enc = raw.subarray(zero + 1)
|
||||
if (enc.length === 0) return ascii.toString('latin1')
|
||||
|
||||
const out: number[] = []
|
||||
const highByte = enc[0]
|
||||
let encPos = 1
|
||||
let flags = 0
|
||||
let flagBits = 0
|
||||
while (encPos < enc.length) {
|
||||
if (flagBits === 0) {
|
||||
flags = enc[encPos++]
|
||||
flagBits = 8
|
||||
}
|
||||
switch (flags >> 6) {
|
||||
case 0:
|
||||
out.push(enc[encPos++])
|
||||
break
|
||||
case 1:
|
||||
out.push(enc[encPos++] + (highByte << 8))
|
||||
break
|
||||
case 2:
|
||||
out.push(enc[encPos] + (enc[encPos + 1] << 8))
|
||||
encPos += 2
|
||||
break
|
||||
case 3: {
|
||||
let length = enc[encPos++]
|
||||
if (length & 0x80) {
|
||||
const correction = enc[encPos++]
|
||||
for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) {
|
||||
out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8))
|
||||
}
|
||||
} else {
|
||||
for (length += 2; length > 0 && out.length < ascii.length; length--) {
|
||||
out.push(ascii[out.length])
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
flags = (flags << 2) & 0xff
|
||||
flagBits -= 2
|
||||
}
|
||||
return String.fromCharCode(...out)
|
||||
}
|
||||
|
||||
// --- RAR 5.0 ----------------------------------------------------------------
|
||||
|
||||
const R5_MAIN = 1
|
||||
const R5_FILE = 2
|
||||
const R5_ENCRYPTION = 4
|
||||
const R5_END = 5
|
||||
|
||||
/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */
|
||||
function vint(data: Buffer, offset: number): { value: number; next: number } | null {
|
||||
let value = 0
|
||||
let shift = 1
|
||||
for (let i = offset; i < data.length && i < offset + 10; i++) {
|
||||
value += (data[i] & 0x7f) * shift
|
||||
shift *= 128
|
||||
if (!(data[i] & 0x80)) return { value, next: i + 1 }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function parseRar5(data: Buffer, start: number): RarInfo {
|
||||
const info: RarInfo = {
|
||||
format: 'rar5',
|
||||
volume: false,
|
||||
firstVolume: null,
|
||||
solid: false,
|
||||
headersEncrypted: false,
|
||||
entries: [],
|
||||
truncated: true,
|
||||
next: start,
|
||||
}
|
||||
let p = start
|
||||
|
||||
while (p + 5 <= data.length) {
|
||||
const headCrc = data.readUInt32LE(p)
|
||||
const headerSize = vint(data, p + 4)
|
||||
if (!headerSize || headerSize.value === 0) break
|
||||
const headerEnd = headerSize.next + headerSize.value
|
||||
if (headerEnd > data.length) break
|
||||
if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break
|
||||
|
||||
const type = vint(data, headerSize.next)
|
||||
const flags = type && vint(data, type.next)
|
||||
if (!type || !flags) break
|
||||
let cursor = flags.next
|
||||
let extraSize = 0
|
||||
let dataSize = 0
|
||||
if (flags.value & 0x0001) {
|
||||
const v = vint(data, cursor)
|
||||
if (!v) break
|
||||
extraSize = v.value
|
||||
cursor = v.next
|
||||
}
|
||||
if (flags.value & 0x0002) {
|
||||
const v = vint(data, cursor)
|
||||
if (!v) break
|
||||
dataSize = v.value
|
||||
cursor = v.next
|
||||
}
|
||||
|
||||
if (type.value === R5_END) {
|
||||
info.truncated = false
|
||||
p = headerEnd
|
||||
break
|
||||
}
|
||||
if (type.value === R5_ENCRYPTION) {
|
||||
info.headersEncrypted = true
|
||||
break
|
||||
}
|
||||
|
||||
if (type.value === R5_MAIN) {
|
||||
const archiveFlags = vint(data, cursor)
|
||||
if (!archiveFlags) break
|
||||
info.volume = Boolean(archiveFlags.value & 0x0001)
|
||||
// A volume number is only stored for volumes after the first.
|
||||
info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true
|
||||
info.solid = Boolean(archiveFlags.value & 0x0004)
|
||||
} else if (type.value === R5_FILE) {
|
||||
const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize)
|
||||
if (!entry) break
|
||||
info.entries.push(entry)
|
||||
}
|
||||
|
||||
const next = headerEnd + dataSize
|
||||
if (next <= p) break
|
||||
p = next
|
||||
}
|
||||
info.next = p
|
||||
return info
|
||||
}
|
||||
|
||||
function parseRar5File(
|
||||
data: Buffer,
|
||||
start: number,
|
||||
extraStart: number,
|
||||
headerEnd: number,
|
||||
headerFlags: number,
|
||||
dataSize: number
|
||||
): RarEntry | null {
|
||||
const fileFlags = vint(data, start)
|
||||
const unpacked = fileFlags && vint(data, fileFlags.next)
|
||||
const attributes = unpacked && vint(data, unpacked.next)
|
||||
if (!fileFlags || !unpacked || !attributes) return null
|
||||
let cursor = attributes.next
|
||||
if (fileFlags.value & 0x0002) cursor += 4 // mtime
|
||||
if (fileFlags.value & 0x0004) cursor += 4 // data CRC32
|
||||
const compression = vint(data, cursor)
|
||||
const hostOs = compression && vint(data, compression.next)
|
||||
const nameLength = hostOs && vint(data, hostOs.next)
|
||||
if (!compression || !hostOs || !nameLength) return null
|
||||
const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8')
|
||||
// Compression information: bits 7-9 are the method, 0 is "store".
|
||||
const method = Math.floor(compression.value / 128) & 0x7
|
||||
|
||||
// Extra area records: size, type, data. Type 1 is file encryption.
|
||||
let encrypted = false
|
||||
let e = extraStart
|
||||
while (e < headerEnd) {
|
||||
const recordSize = vint(data, e)
|
||||
const recordType = recordSize && vint(data, recordSize.next)
|
||||
if (!recordSize || !recordType || recordSize.value === 0) break
|
||||
if (recordType.value === 0x01) encrypted = true
|
||||
e = recordSize.next + recordSize.value
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
size: fileFlags.value & 0x0008 ? null : unpacked.value,
|
||||
packedSize: dataSize,
|
||||
directory: Boolean(fileFlags.value & 0x0001),
|
||||
encrypted,
|
||||
splitBefore: Boolean(headerFlags & 0x0008),
|
||||
splitAfter: Boolean(headerFlags & 0x0010),
|
||||
stored: method === 0,
|
||||
dataOffset: headerEnd,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The data of an entry stored without compression, copied out of the
|
||||
* archive buffer. Null when it is compressed, encrypted, split over volumes
|
||||
* or not completely in the buffer.
|
||||
*/
|
||||
export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null {
|
||||
if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
||||
if (entry.size === null || entry.size !== entry.packedSize) return null
|
||||
const end = entry.dataOffset + entry.packedSize
|
||||
if (end > data.length) return null
|
||||
return data.subarray(entry.dataOffset, end)
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '../yenc/index.js'
|
||||
import type { VolumeReader } from './walk.js'
|
||||
|
||||
/** Downloads one article body (on a pooled connection, or through the seed API's relay). */
|
||||
export type BodyFetcher = (messageId: string) => Promise<Buffer>
|
||||
|
||||
/**
|
||||
* Reads volumes (each a list of segment message ids, in order) a segment at
|
||||
* a time, finding the segment for an offset from the part size (yEnc
|
||||
* posters use one size per set) and correcting from the `=ypart` offsets
|
||||
* when a guess is off. Stops reading after `maxSegments` downloads.
|
||||
*/
|
||||
export class SegmentReader implements VolumeReader {
|
||||
fetched = 0
|
||||
private partSize = 0
|
||||
private readonly sizes = new Map<number, number>()
|
||||
private readonly cache = new Map<string, DecodedSegment>()
|
||||
|
||||
constructor(
|
||||
private readonly files: { id: string }[][],
|
||||
private readonly maxSegments: number,
|
||||
private readonly fetchBody: BodyFetcher
|
||||
) {}
|
||||
|
||||
get volumes() {
|
||||
return this.files.length
|
||||
}
|
||||
|
||||
size(v: number) {
|
||||
return this.sizes.get(v) ?? null
|
||||
}
|
||||
|
||||
async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
|
||||
const count = this.files[v].length
|
||||
let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
|
||||
for (let tries = 0; tries < 4; tries++) {
|
||||
const segment = await this.segment(v, index)
|
||||
if (!segment) return null
|
||||
const end = segment.offset + segment.data.length
|
||||
if (offset >= segment.offset && offset < end) {
|
||||
const data = segment.data.subarray(offset - segment.offset)
|
||||
if (!more || index + 1 >= count) return data
|
||||
const next = await this.segment(v, index + 1)
|
||||
return next ? Buffer.concat([data, next.data]) : null
|
||||
}
|
||||
const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
|
||||
const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
|
||||
if (moved === index) return null
|
||||
index = moved
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private async segment(v: number, index: number): Promise<DecodedSegment | null> {
|
||||
const key = `${v}:${index}`
|
||||
const cached = this.cache.get(key)
|
||||
if (cached) return cached
|
||||
const part = this.files[v][index]
|
||||
if (!part?.id || this.fetched >= this.maxSegments) return null
|
||||
this.fetched++
|
||||
let body: Buffer
|
||||
try {
|
||||
body = await this.fetchBody(part.id)
|
||||
} catch {
|
||||
// A missing article (or the relay failing) ends the walk with what was found.
|
||||
return null
|
||||
}
|
||||
let segment: DecodedSegment
|
||||
try {
|
||||
segment = decodeSegment(body)
|
||||
} catch (error) {
|
||||
if (!(error instanceof YencError) || error instanceof YencCrcError) return null
|
||||
segment = decodeSegment(body, { lenientSize: true })
|
||||
}
|
||||
this.sizes.set(v, segment.meta.begin.size)
|
||||
if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
|
||||
// Walks move forward: the last few segments are enough.
|
||||
if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
|
||||
this.cache.set(key, segment)
|
||||
return segment
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import { parseRarHeaders, rarFormat, type RarEntry, type RarInfo } from './parse.js'
|
||||
|
||||
/** Reads the volumes of a RAR set, a segment at a time. */
|
||||
export interface VolumeReader {
|
||||
readonly volumes: number
|
||||
/**
|
||||
* Bytes of volume `v` from `offset` to the end of the segment that holds
|
||||
* it; with `more`, the following segment too (for a header cut off by the
|
||||
* segment end). Null when that can't be read (missing article, budget).
|
||||
*/
|
||||
read(v: number, offset: number, more: boolean): Promise<Buffer | null>
|
||||
/** Size of volume `v`, known once something of it was read. */
|
||||
size(v: number): number | null
|
||||
}
|
||||
|
||||
export interface RarWalk {
|
||||
format: 'rar4' | 'rar5' | null
|
||||
/** One entry per name, from its first header. */
|
||||
entries: RarEntry[]
|
||||
/** Every volume was walked to its end. */
|
||||
complete: boolean
|
||||
headersEncrypted: boolean
|
||||
/** Why the walk ended early. */
|
||||
stopped?: 'unreadable' | 'invalid' | 'encrypted'
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists a RAR set completely by hopping from header to header (CONCEPT.md
|
||||
* D40). Each header says how much data follows it, so the next header's
|
||||
* offset is known and only the segment holding it is read: one segment per
|
||||
* entry and one per volume, instead of the whole set. Split files repeat
|
||||
* their header at the start of every volume they continue in.
|
||||
*/
|
||||
export async function walkRar(reader: VolumeReader): Promise<RarWalk> {
|
||||
const seen = new Map<string, RarEntry>()
|
||||
const result = (stopped?: RarWalk['stopped']): RarWalk => ({
|
||||
format,
|
||||
entries: [...seen.values()],
|
||||
complete: !stopped,
|
||||
headersEncrypted: stopped === 'encrypted',
|
||||
stopped,
|
||||
})
|
||||
let format: 'rar4' | 'rar5' | null = null
|
||||
let v = 0
|
||||
let offset = 0
|
||||
|
||||
while (v < reader.volumes) {
|
||||
let info: RarInfo | null = null
|
||||
let reached = -1
|
||||
for (const more of [false, true]) {
|
||||
const data = await reader.read(v, offset, more)
|
||||
if (!data) return result('unreadable')
|
||||
if (offset === 0) format = rarFormat(data) ?? format
|
||||
info = offset === 0 ? parseRarHeaders(data) : format && parseRarHeaders(data, format)
|
||||
if (!info) return result('invalid')
|
||||
// Stopped inside the data: a header the segment end cut off (read on), or a bad one.
|
||||
if (!info.truncated || info.next >= data.length || info.headersEncrypted) break
|
||||
// Read on and got further: the next header is cut off at the end of the following segment.
|
||||
if (more && info.next > reached) break
|
||||
if (more) return result('invalid')
|
||||
reached = info.next
|
||||
}
|
||||
if (info!.headersEncrypted) return result('encrypted')
|
||||
for (const entry of info!.entries) if (!seen.has(entry.name)) seen.set(entry.name, entry)
|
||||
|
||||
const next = offset + info!.next
|
||||
const size = reader.size(v)
|
||||
// The end-of-archive header closes every volume; data running to the end does too.
|
||||
if (!info!.truncated || size === null || next >= size) {
|
||||
v++
|
||||
offset = 0
|
||||
} else {
|
||||
offset = next
|
||||
}
|
||||
}
|
||||
return result()
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
/**
|
||||
* Grouping of files into releases (downloads), independent of AdonisJS.
|
||||
*/
|
||||
export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, isGenericName, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'
|
||||
@@ -1,223 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { looksLikeFilename } from '../subject/file_key.js'
|
||||
|
||||
export interface ReleaseIdentity {
|
||||
key: string
|
||||
/** Human-readable name: the subject text or the shared filename stem. */
|
||||
name: string
|
||||
/** Number of files announced by the `[i/M]` counter, when present. */
|
||||
fileTotal?: number
|
||||
/** Which rule recognised the release. */
|
||||
source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot'
|
||||
}
|
||||
|
||||
export interface ReleaseInput {
|
||||
poster: string
|
||||
/** Subject without segment counter (the binary's `subject`). */
|
||||
subject: string
|
||||
/** Filename of the binary. */
|
||||
name: string
|
||||
/** True when the identity came from the yEnc body (obfuscated post). */
|
||||
weak: boolean
|
||||
}
|
||||
|
||||
// Suffixes that differ between the files of one release. Stripped repeatedly,
|
||||
// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x".
|
||||
const SUFFIXES = [
|
||||
/\.vol\d+[+-]\d+\.par2$/i,
|
||||
/\.par2$/i,
|
||||
/\.part\d+\.rar$/i,
|
||||
/\.r\d{2,3}$/i,
|
||||
/\.rar$/i,
|
||||
/\.7z(\.\d{3})?$/i,
|
||||
/\.zip(\.\d{3})?$/i,
|
||||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i,
|
||||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||||
/\.tar$/i,
|
||||
/\.(zst|gz|xz|bz2)$/i,
|
||||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||||
]
|
||||
|
||||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||||
export function fileStem(name: string): string {
|
||||
let stem = name.trim()
|
||||
let changed = true
|
||||
while (changed) {
|
||||
changed = false
|
||||
for (const suffix of SUFFIXES) {
|
||||
const next = stem.replace(suffix, '')
|
||||
if (next !== stem && next.length > 0) {
|
||||
stem = next
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return stem
|
||||
}
|
||||
|
||||
/**
|
||||
* What remains of a subject once the filename, counters, "yEnc" and sizes are
|
||||
* removed. Posters that name the release outside the quotes put it here.
|
||||
*/
|
||||
export function subjectRemainder(subject: string): string {
|
||||
return subject
|
||||
.replace(/"[^"]*"/g, ' ')
|
||||
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
|
||||
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
|
||||
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
|
||||
.replace(/\byEnc\b/gi, ' ')
|
||||
.replace(/[\s\-–.:|]+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Posting tools put random per-file tokens and poster IDs in subjects (e.g.
|
||||
* `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
|
||||
* name once those are removed identifies a release.
|
||||
*/
|
||||
function isRandomToken(token: string): boolean {
|
||||
const t = token.replace(/^\[|\]$/g, '')
|
||||
if (t.length < 6) return false
|
||||
if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true
|
||||
return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t)
|
||||
}
|
||||
|
||||
/**
|
||||
* A stable per-post token: one short run of letters/digits with at least one
|
||||
* digit. Requiring a digit keeps constant labels like `[PRiVATE]` from
|
||||
* grouping different posts of the same poster.
|
||||
*/
|
||||
export function isPostToken(remainder: string): boolean {
|
||||
const compact = remainder.replace(/[\s\[\]()_-]+/g, '')
|
||||
return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact)
|
||||
}
|
||||
|
||||
export function namesRelease(remainder: string): boolean {
|
||||
const words = remainder
|
||||
.split(/\s+/)
|
||||
.filter((w) => w && !isRandomToken(w))
|
||||
.join(' ')
|
||||
// At least two words of letters, joined the way release names are.
|
||||
return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words)
|
||||
}
|
||||
|
||||
function hash(...parts: (string | number | undefined)[]) {
|
||||
return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides which release (download) a file belongs to. Returns null when the
|
||||
* file cannot be grouped from its name alone (random extensionless names in
|
||||
* obfuscated posts; those need par2 matching). See CONCEPT.md (D16).
|
||||
*/
|
||||
export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
||||
const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
|
||||
const stem = fileStem(input.name)
|
||||
|
||||
if (input.weak) {
|
||||
// Obfuscating posters randomise the poster per article; a real filename
|
||||
// stem is specific enough on its own.
|
||||
if (!looksLikeFilename(input.name)) return null
|
||||
return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
|
||||
}
|
||||
|
||||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||||
const remainder = subjectRemainder(input.subject)
|
||||
if (namesRelease(remainder)) {
|
||||
return {
|
||||
key: hash('subject', input.poster, remainder),
|
||||
name: remainder,
|
||||
fileTotal,
|
||||
source: 'subject',
|
||||
}
|
||||
}
|
||||
// Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
|
||||
// [i/M] counter it groups the files of one post whose names are random.
|
||||
if (fileTotal && isPostToken(remainder)) {
|
||||
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
|
||||
}
|
||||
// The [i/M] total is informational, not part of the key: posters often
|
||||
// upload the par2 index without a counter, and it belongs to the same post.
|
||||
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
|
||||
}
|
||||
|
||||
/** Release of a par2 recovery set; every file listed in it belongs to it. */
|
||||
export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity {
|
||||
return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' }
|
||||
}
|
||||
|
||||
const SEPARATOR = /[._ -]/
|
||||
|
||||
/**
|
||||
* A release name from the files a par2 set lists. Files that differ only in
|
||||
* the middle (episodes S03E01…S03E20, parts) give their shared name, with
|
||||
* the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to
|
||||
* SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato.
|
||||
* Otherwise the longest stem, which tends to be the most descriptive (the
|
||||
* video file).
|
||||
*/
|
||||
export function par2ReleaseName(fileNames: string[]): string | undefined {
|
||||
const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean)
|
||||
if (stems.length === 0) return undefined
|
||||
const longest = [...stems].sort((a, b) => b.length - a.length)[0]
|
||||
if (stems.length < 2) return longest
|
||||
|
||||
let prefix = stems.reduce((p, s) => {
|
||||
let i = 0
|
||||
while (i < p.length && i < s.length && p[i] === s[i]) i++
|
||||
return p.slice(0, i)
|
||||
})
|
||||
let suffix = stems.reduce((p, s) => {
|
||||
let i = 0
|
||||
while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++
|
||||
return p.slice(p.length - i)
|
||||
})
|
||||
const shortest = Math.min(...stems.map((s) => s.length))
|
||||
if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest)
|
||||
|
||||
// Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token.
|
||||
const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix)
|
||||
if (season) prefix = season[1]
|
||||
else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) {
|
||||
prefix = prefix.replace(/[^._ -]*$/, '')
|
||||
}
|
||||
suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '')
|
||||
prefix = prefix.replace(/[._ -]+$/, '')
|
||||
|
||||
const name = suffix ? prefix + suffix : prefix
|
||||
// Only when the shared part carries most of the name (not just "The.").
|
||||
return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest
|
||||
}
|
||||
|
||||
/**
|
||||
* Release of a RAR volume set, from what its headers list: the shared
|
||||
* top-level directory if there is one, otherwise the first entry. For
|
||||
* volumes after the first, the first entry is the file that continues from
|
||||
* the previous volume, so all volumes of a single-file archive agree.
|
||||
*/
|
||||
/**
|
||||
* Folder and file names that say nothing about what a release is: disc
|
||||
* structures (VIDEO_TS on every DVD), samples, subtitles, disc numbers.
|
||||
* Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export const GENERIC_NAMES = /^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?\d{1,2})$/i
|
||||
|
||||
export function isGenericName(name: string): boolean {
|
||||
return GENERIC_NAMES.test(name.trim())
|
||||
}
|
||||
|
||||
export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
|
||||
if (entries.length === 0) return null
|
||||
const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
const nested = entries.some((e) => /[\\/]/.test(e.name))
|
||||
// A generic top folder (VIDEO_TS) is shared by unrelated archives: key on the first entry instead.
|
||||
if (nested && tops.size === 1 && !isGenericName([...tops][0])) {
|
||||
const [top] = tops
|
||||
return { key: hash('rar-dir', top), name: top, source: 'rar' }
|
||||
}
|
||||
const first = entries[0]
|
||||
const base = first.name.split(/[\\/]/).pop() ?? first.name
|
||||
return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' }
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
/**
|
||||
* Seed planning arithmetic (CONCEPT.md D38), independent of the database.
|
||||
*/
|
||||
|
||||
/** The parts of `from`..`to` (inclusive) that no range in `covered` (inclusive pairs) overlaps, oldest first. */
|
||||
export function uncovered(from: number, to: number, covered: [number, number][]): [number, number][] {
|
||||
const gaps: [number, number][] = []
|
||||
let next = from
|
||||
for (const [start, end] of [...covered].sort((a, b) => a[0] - b[0])) {
|
||||
if (end < next) continue
|
||||
if (start > to) break
|
||||
if (start > next) gaps.push([next, Math.min(start - 1, to)])
|
||||
next = Math.max(next, end + 1)
|
||||
if (next > to) return gaps
|
||||
}
|
||||
if (next <= to) gaps.push([next, to])
|
||||
return gaps
|
||||
}
|
||||
|
||||
/** `gaps` cut into ranges of at most `size` articles, newest first (from the top of the newest gap down). */
|
||||
export function cut(gaps: [number, number][], size: number): [number, number][] {
|
||||
const ranges: [number, number][] = []
|
||||
for (const [from, to] of [...gaps].sort((a, b) => b[1] - a[1])) {
|
||||
for (let end = to; end >= from; end -= size) ranges.push([Math.max(from, end - size + 1), end])
|
||||
}
|
||||
return ranges
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
import type { Overview } from '../nntp/xover_client.js'
|
||||
import { segmentFromHeader, type IngestSegment, type OverviewHeader } from '../ingest/segment.js'
|
||||
import { FileAggregator } from '../aggregate/index.js'
|
||||
import { toFragment, toSeedFile, type SeedBatch } from './wire.js'
|
||||
|
||||
/** Where a range's overviews come from: an NNTP connection, or the seed API's relay. */
|
||||
export interface OverviewSource {
|
||||
group(name: string): Promise<unknown>
|
||||
xover(from: number, to: number): Promise<Overview[]>
|
||||
}
|
||||
|
||||
export interface RunRangeOptions {
|
||||
client: OverviewSource
|
||||
group: string
|
||||
/** Inclusive article numbers. */
|
||||
start: number
|
||||
end: number
|
||||
/** Articles per XOVER. */
|
||||
chunk?: number
|
||||
/** Batches without a new part before an open file leaves as fragments. */
|
||||
idle?: number
|
||||
/** XOVER batches per SeedBatch handed to `sink`. */
|
||||
submitEvery?: number
|
||||
/** Headers left to the deferred-body path (counted, not indexed). */
|
||||
isDeferred?: (header: OverviewHeader) => boolean
|
||||
/** Receives batches in order; the next XOVER is already in flight meanwhile. */
|
||||
sink: (batch: SeedBatch) => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Indexes one article range on a seeding node (CONCEPT.md D37): XOVER in
|
||||
* chunks over one connection (the next chunk in flight while the previous
|
||||
* one is handled), headers parsed with the live rules, grouped into files
|
||||
* in memory; complete files and given-up fragments go to `sink` in order.
|
||||
* Shared by seed:range (stores directly) and seed:node (submits over HTTPS).
|
||||
*/
|
||||
export async function runRange(options: RunRangeOptions): Promise<void> {
|
||||
const chunk = options.chunk ?? 20_000
|
||||
const idle = options.idle ?? 5
|
||||
const submitEvery = options.submitEvery ?? 5
|
||||
const aggregator = new FileAggregator<IngestSegment>()
|
||||
// A range fully reported before but not finished (its node went away right at the end) resumes past its end.
|
||||
if (options.start > options.end) {
|
||||
await options.sink(emptyBatch(options.end + 1))
|
||||
return
|
||||
}
|
||||
await options.client.group(options.group)
|
||||
|
||||
const fetch = (from: number) => {
|
||||
const promise = options.client.xover(from, Math.min(from + chunk - 1, options.end))
|
||||
// Awaited after the current batch is handled; until then a failure must not count as unhandled.
|
||||
promise.catch(() => {})
|
||||
return promise
|
||||
}
|
||||
|
||||
let batch = emptyBatch(options.start)
|
||||
let pending = 0
|
||||
let from = options.start
|
||||
let next: Promise<Overview[]> | null = fetch(from)
|
||||
while (next) {
|
||||
const overviews = await next
|
||||
from += chunk
|
||||
next = from <= options.end ? fetch(from) : null
|
||||
|
||||
const segments: IngestSegment[] = []
|
||||
for (const [articleNumber, header] of overviews) {
|
||||
if (!Number.isFinite(articleNumber) || !header['message-id']) continue
|
||||
const segment = segmentFromHeader(header, options.group, articleNumber)
|
||||
if (segment) segments.push(segment)
|
||||
else if (options.isDeferred?.(header)) batch.deferred++
|
||||
else batch.obfuscated++
|
||||
}
|
||||
batch.headers += overviews.length
|
||||
batch.files.push(...aggregator.add(segments).map(toSeedFile))
|
||||
batch.fragments.push(...aggregator.evict(idle).map(toFragment))
|
||||
batch.next = Math.min(from, options.end + 1)
|
||||
|
||||
if (++pending >= submitEvery || !next) {
|
||||
if (!next) batch.fragments.push(...aggregator.evict(0).map(toFragment))
|
||||
await options.sink(batch)
|
||||
batch = emptyBatch(batch.next)
|
||||
pending = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function emptyBatch(next: number): SeedBatch {
|
||||
return { next, headers: 0, obfuscated: 0, deferred: 0, files: [], fragments: [] }
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import type { IngestSegment } from '../ingest/segment.js'
|
||||
import type { ReleaseIdentity } from '../release/index.js'
|
||||
import type { AggregatedFile } from '../aggregate/index.js'
|
||||
|
||||
/**
|
||||
* A complete file as a seeding node reports it (CONCEPT.md D37): what the
|
||||
* file, its release and its binary need, once per file, and the segments
|
||||
* as [part number, message-id, bytes].
|
||||
*/
|
||||
export interface SeedFile {
|
||||
fileKey: string
|
||||
name: string
|
||||
poster: string
|
||||
subject: string
|
||||
totalParts: number
|
||||
groups: string[]
|
||||
/** Earliest post date of its articles, ISO 8601. */
|
||||
postedAt: string | null
|
||||
release: ReleaseIdentity | null
|
||||
segments: [number, string, number][]
|
||||
}
|
||||
|
||||
/** A segment of a file that didn't complete on the node, for the regular ingest path. */
|
||||
export type SeedFragment = Omit<IngestSegment, 'postedAt' | 'articleNumber'> & { postedAt?: string; articleNumber?: number }
|
||||
|
||||
/** What a node sends per batch: the article it has reached, and what it found up to there. */
|
||||
export interface SeedBatch {
|
||||
/** First article number not covered yet (everything below it, from the range start, is in this or earlier batches). */
|
||||
next: number
|
||||
headers: number
|
||||
obfuscated: number
|
||||
deferred: number
|
||||
files: SeedFile[]
|
||||
fragments: SeedFragment[]
|
||||
}
|
||||
|
||||
export function toSeedFile(file: AggregatedFile<IngestSegment>): SeedFile {
|
||||
const first = file.segments[0]
|
||||
const times = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
|
||||
return {
|
||||
fileKey: file.fileKey,
|
||||
name: first.name,
|
||||
poster: first.poster,
|
||||
subject: first.subject,
|
||||
totalParts: first.totalParts,
|
||||
groups: first.groups,
|
||||
postedAt: times.length > 0 ? new Date(Math.min(...times)).toISOString() : null,
|
||||
release: first.release ?? null,
|
||||
segments: file.segments.map((s) => [s.number, s.messageId, s.bytes]),
|
||||
}
|
||||
}
|
||||
|
||||
export function toFragment(segment: IngestSegment): SeedFragment {
|
||||
const { postedAt, articleNumber, ...rest } = segment
|
||||
const date = postedAt && !Number.isNaN(postedAt.getTime()) ? postedAt.toISOString() : undefined
|
||||
return { ...rest, postedAt: date, articleNumber: articleNumber === undefined ? undefined : Number(articleNumber) }
|
||||
}
|
||||
|
||||
export function fromFragment(fragment: SeedFragment): IngestSegment {
|
||||
// The JSON body parser turns empty strings into null; the columns want ''.
|
||||
return {
|
||||
...fragment,
|
||||
poster: fragment.poster ?? '',
|
||||
subject: fragment.subject ?? '',
|
||||
name: fragment.name ?? '',
|
||||
postedAt: fragment.postedAt ? new Date(fragment.postedAt) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/** A file from a node's JSON, with empty strings back where the body parser made them null. */
|
||||
export function normalizeSeedFile(file: SeedFile): SeedFile {
|
||||
return { ...file, name: file.name ?? '', poster: file.poster ?? '', subject: file.subject ?? '' }
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* 7-Zip archives, independent of AdonisJS. A 7z archive starts with a
|
||||
* 32-byte signature header that points to the archive's header (the file
|
||||
* list) at its very end, often LZMA-compressed ("encoded") with its packed
|
||||
* data just before it. So the list needs the first 32 bytes and the tail:
|
||||
* for a split archive (.7z.001, .002, …), the start of the first volume and
|
||||
* the end of the last. Listing itself is left to libarchive (bsdtar).
|
||||
*/
|
||||
|
||||
const SIGNATURE = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])
|
||||
const SIGNATURE_HEADER_SIZE = 32
|
||||
/** Header property ids (7zFormat.txt). */
|
||||
const K_END = 0x00
|
||||
const K_PACK_INFO = 0x06
|
||||
const K_UNPACK_INFO = 0x07
|
||||
const K_SIZE = 0x09
|
||||
const K_CRC = 0x0a
|
||||
const K_FOLDER = 0x0b
|
||||
const K_ENCODED_HEADER = 0x17
|
||||
|
||||
export interface SevenZipStart {
|
||||
/** Where the archive header starts, from the start of the archive. */
|
||||
headerStart: number
|
||||
headerSize: number
|
||||
headerCrc: number
|
||||
/** The archive's total size (the header is its last part). */
|
||||
archiveSize: number
|
||||
}
|
||||
|
||||
/** Reads the signature header; null when `data` isn't a 7z start or its CRC fails. */
|
||||
export function parseSevenZipStart(data: Buffer): SevenZipStart | null {
|
||||
if (data.length < SIGNATURE_HEADER_SIZE || !data.subarray(0, 6).equals(SIGNATURE)) return null
|
||||
if ((crc32(data.subarray(12, 32)) >>> 0) !== data.readUInt32LE(8)) return null
|
||||
const offset = Number(data.readBigUInt64LE(12))
|
||||
const size = Number(data.readBigUInt64LE(20))
|
||||
const headerStart = SIGNATURE_HEADER_SIZE + offset
|
||||
return { headerStart, headerSize: size, headerCrc: data.readUInt32LE(28), archiveSize: headerStart + size }
|
||||
}
|
||||
|
||||
/** Whether `header` (the bytes at headerStart) is the archive header its signature header describes. */
|
||||
export function checkHeader(start: SevenZipStart, header: Buffer): boolean {
|
||||
return header.length === start.headerSize && (crc32(header) >>> 0) === start.headerCrc
|
||||
}
|
||||
|
||||
/**
|
||||
* Where the bytes needed for listing start: the header itself, or for an
|
||||
* encoded header the packed (compressed) header stream it points to.
|
||||
*/
|
||||
export function listingStart(start: SevenZipStart, header: Buffer): number {
|
||||
if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return start.headerStart
|
||||
const [packPos] = readNumber(header, 2)
|
||||
return SIGNATURE_HEADER_SIZE + packPos
|
||||
}
|
||||
|
||||
/** 7zAES, the coder of password-protected streams. */
|
||||
const AES_CODER = Buffer.from([0x06, 0xf1, 0x07, 0x01])
|
||||
|
||||
/**
|
||||
* Whether an encoded header (`header`, the bytes at headerStart) is
|
||||
* encrypted (7z -mhe): its coders include 7zAES. Then nothing can be listed
|
||||
* without the password, and neither the packed header nor bsdtar is needed.
|
||||
* Null when the header isn't encoded or doesn't read as expected.
|
||||
*/
|
||||
export function encodedHeaderEncrypted(header: Buffer): boolean | null {
|
||||
try {
|
||||
if (header[0] !== K_ENCODED_HEADER || header[1] !== K_PACK_INFO) return null
|
||||
let pos = 2
|
||||
let numPackStreams: number
|
||||
;[, pos] = readNumber(header, pos) // packPos
|
||||
;[numPackStreams, pos] = readNumber(header, pos)
|
||||
for (let id = header[pos++]; id !== K_END; id = header[pos++]) {
|
||||
if (id === K_SIZE) {
|
||||
for (let i = 0; i < numPackStreams; i++) [, pos] = readNumber(header, pos)
|
||||
} else if (id === K_CRC) {
|
||||
const allDefined = header[pos++]
|
||||
let defined = numPackStreams
|
||||
if (!allDefined) {
|
||||
const bits = header.subarray(pos, pos + Math.ceil(numPackStreams / 8))
|
||||
pos += bits.length
|
||||
defined = [...bits].reduce((sum, byte) => sum + byte.toString(2).replaceAll('0', '').length, 0)
|
||||
}
|
||||
pos += 4 * defined
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
if (pos >= header.length) return null
|
||||
}
|
||||
if (header[pos++] !== K_UNPACK_INFO || header[pos++] !== K_FOLDER) return null
|
||||
let numFolders: number
|
||||
;[numFolders, pos] = readNumber(header, pos)
|
||||
if (numFolders < 1 || header[pos++] !== 0) return null // folders stored elsewhere
|
||||
let numCoders: number
|
||||
;[numCoders, pos] = readNumber(header, pos)
|
||||
if (!(numCoders >= 1) || pos >= header.length) return null
|
||||
for (let c = 0; c < numCoders; c++) {
|
||||
const flags = header[pos++]
|
||||
const idSize = flags & 0x0f
|
||||
if (header.subarray(pos, pos + idSize).equals(AES_CODER)) return true
|
||||
pos += idSize
|
||||
if (flags & 0x10) {
|
||||
;[, pos] = readNumber(header, pos)
|
||||
;[, pos] = readNumber(header, pos)
|
||||
}
|
||||
if (flags & 0x20) {
|
||||
let size: number
|
||||
;[size, pos] = readNumber(header, pos)
|
||||
pos += size
|
||||
}
|
||||
if (pos > header.length) return null
|
||||
}
|
||||
return false
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 7z's variable-length number: the leading 1-bits of the first byte count
|
||||
* the little-endian bytes that follow; its remaining bits are the high part.
|
||||
*/
|
||||
export function readNumber(data: Buffer, pos: number): [value: number, next: number] {
|
||||
const first = data[pos]
|
||||
let mask = 0x80
|
||||
let value = 0
|
||||
for (let i = 0; i < 8; i++) {
|
||||
if ((first & mask) === 0) {
|
||||
value += (first & (mask - 1)) * 2 ** (8 * i)
|
||||
return [value, pos + 1 + i]
|
||||
}
|
||||
value += data[pos + 1 + i] * 2 ** (8 * i)
|
||||
mask >>= 1
|
||||
}
|
||||
return [value, pos + 9]
|
||||
}
|
||||
|
||||
export interface SevenZipEntry {
|
||||
name: string
|
||||
size: number
|
||||
directory: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Entries from `bsdtar -tv` output, e.g.
|
||||
* `-rw-rw-r-- 0 0 0 3000000 Sep 30 19:13 Some.Release/movie.mkv`.
|
||||
*/
|
||||
export function parseBsdtarListing(stdout: string): SevenZipEntry[] {
|
||||
const entries: SevenZipEntry[] = []
|
||||
for (const line of stdout.split('\n')) {
|
||||
const match = /^([-dlbcps])\S*\s+\d+\s+\S+\s+\S+\s+(\d+)\s+\w{3}\s+\d{1,2}\s+(?:\d{1,2}:\d{2}|\d{4})\s(.+)$/.exec(line)
|
||||
if (!match) continue
|
||||
const directory = match[1] === 'd'
|
||||
entries.push({ name: directory ? match[3].replace(/\/$/, '') : match[3], size: Number(match[2]), directory })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
/** Volume number of a split 7z name (`x.7z.001` → 1), 0 for an unsplit `.7z`, null otherwise. */
|
||||
export function sevenZipVolume(name: string): number | null {
|
||||
const split = /\.7z\.(\d{1,4})$/i.exec(name)
|
||||
if (split) return Number(split[1])
|
||||
return /\.7z$/i.test(name) ? 0 : null
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { mkdtemp, open, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import { decodeSegment, type DecodedSegment } from '../yenc/index.js'
|
||||
import type { RarInfo } from '../rar/parse.js'
|
||||
import type { BodyFetcher } from '../rar/segment_reader.js'
|
||||
import {
|
||||
checkHeader,
|
||||
encodedHeaderEncrypted,
|
||||
listingStart,
|
||||
parseBsdtarListing,
|
||||
parseSevenZipStart,
|
||||
type SevenZipStart,
|
||||
} from './index.js'
|
||||
|
||||
const run = promisify(execFile)
|
||||
|
||||
/** The tail read for a listing: header plus packed header; more is not a header. */
|
||||
const MAX_TAIL_BYTES = 16 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* Lists a 7z archive, also a split one (.7z.001, .002, …), from the first
|
||||
* segment of the first volume (`firstData`, decoded) and the last segment(s)
|
||||
* of the last one (CONCEPT.md D19). Those bytes are put at their place in a
|
||||
* sparse file the size of the whole archive, which bsdtar (libarchive)
|
||||
* lists: it only reads the start and the header at the end. Null when it
|
||||
* isn't a 7z start, or the tail doesn't hold the header its start points to.
|
||||
*/
|
||||
export async function listSevenZip(
|
||||
firstData: Buffer,
|
||||
volumeCount: number,
|
||||
lastVolume: { number: number; id: string }[],
|
||||
fetch: BodyFetcher,
|
||||
warn: (message: string) => void = () => {}
|
||||
): Promise<RarInfo | null> {
|
||||
const start = parseSevenZipStart(firstData)
|
||||
if (!start) return null
|
||||
const split = volumeCount > 1
|
||||
if (start.headerSize === 0) return sevenZipInfo([], false, false, split)
|
||||
|
||||
const tail = await readTail(lastVolume, start, fetch)
|
||||
if (!tail) return null
|
||||
// Encrypted headers (-mhe): known from the header alone, no listing possible (most 7z uploads).
|
||||
if ('headersEncrypted' in tail) return sevenZipInfo([], true, true, split)
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
|
||||
try {
|
||||
const archive = path.join(dir, 'archive.7z')
|
||||
const handle = await open(archive, 'w')
|
||||
try {
|
||||
await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
|
||||
await handle.write(tail.data, 0, tail.data.length, tail.from)
|
||||
await handle.truncate(start.archiveSize)
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return await bsdtar(archive, split, warn)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
|
||||
async function bsdtar(archive: string, split: boolean, warn: (message: string) => void): Promise<RarInfo | null> {
|
||||
let stdout = ''
|
||||
try {
|
||||
stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
|
||||
} catch (error: any) {
|
||||
if (/header is encrypted/i.test(String(error.stderr))) return sevenZipInfo([], true, true, split)
|
||||
warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
|
||||
return null
|
||||
}
|
||||
const entries = parseBsdtarListing(stdout)
|
||||
const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
|
||||
let encrypted = false
|
||||
if (sample) {
|
||||
try {
|
||||
await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
|
||||
} catch (error: any) {
|
||||
// Without the data (it's all zeros here), anything else fails too: only this message counts.
|
||||
encrypted = /content is encrypted/i.test(String(error.stderr))
|
||||
}
|
||||
}
|
||||
return sevenZipInfo(entries, false, encrypted, split)
|
||||
}
|
||||
|
||||
export function sevenZipInfo(
|
||||
entries: { name: string; size: number; directory: boolean }[],
|
||||
headersEncrypted: boolean,
|
||||
encrypted: boolean,
|
||||
split: boolean
|
||||
): RarInfo {
|
||||
return {
|
||||
format: '7z',
|
||||
volume: split,
|
||||
firstVolume: true,
|
||||
solid: false,
|
||||
headersEncrypted,
|
||||
truncated: false,
|
||||
next: 0,
|
||||
entries: entries.map((e) => ({
|
||||
name: e.name,
|
||||
size: e.size,
|
||||
packedSize: 0,
|
||||
directory: e.directory,
|
||||
encrypted: encrypted && !e.directory,
|
||||
splitBefore: false,
|
||||
splitAfter: false,
|
||||
stored: false,
|
||||
dataOffset: 0,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The archive's tail from the last volume: the header (checked against its
|
||||
* CRC) and, for an encoded header, the packed stream before it. `from` is
|
||||
* its position in the whole archive.
|
||||
*/
|
||||
async function readTail(
|
||||
last: { number: number; id: string }[],
|
||||
start: SevenZipStart,
|
||||
fetch: BodyFetcher
|
||||
): Promise<{ from: number; data: Buffer } | { headersEncrypted: true } | null> {
|
||||
const ids = new Map(last.map((s) => [s.number, s]))
|
||||
const numbers = [...ids.keys()]
|
||||
if (numbers.length === 0) return null
|
||||
const segments = new Map<number, DecodedSegment>()
|
||||
const segment = async (n: number) => {
|
||||
if (!segments.has(n)) {
|
||||
const id = ids.get(n)?.id
|
||||
if (!id) throw new Error(`part ${n} of the last 7z volume is missing`)
|
||||
segments.set(n, decodeSegment(await fetch(id), { lenientSize: true }))
|
||||
}
|
||||
return segments.get(n)!
|
||||
}
|
||||
|
||||
const lastNumber = numbers[numbers.length - 1]
|
||||
const lastSegment = await segment(lastNumber)
|
||||
const volumeSize = lastSegment.meta.begin.size
|
||||
const volumeStart = start.archiveSize - volumeSize
|
||||
const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
|
||||
const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
|
||||
if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
|
||||
|
||||
// Bytes [from, to) of the whole archive, from the last volume's parts.
|
||||
const read = async (from: number, to: number): Promise<Buffer | null> => {
|
||||
if (from < volumeStart) return null
|
||||
const a = from - volumeStart
|
||||
const b = to - volumeStart
|
||||
const pieces: Buffer[] = []
|
||||
for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
|
||||
const s = await segment(n)
|
||||
const offset = (s.meta.part?.begin ?? 1) - 1
|
||||
if (offset !== (n - 1) * partSize) return null
|
||||
pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
|
||||
}
|
||||
const data = Buffer.concat(pieces)
|
||||
return data.length === b - a ? data : null
|
||||
}
|
||||
|
||||
const header = await read(start.headerStart, start.archiveSize)
|
||||
if (!header || !checkHeader(start, header)) return null
|
||||
if (encodedHeaderEncrypted(header)) return { headersEncrypted: true }
|
||||
const from = Math.min(listingStart(start, header), start.headerStart)
|
||||
if (start.archiveSize - from > MAX_TAIL_BYTES) return null
|
||||
const data = await read(from, start.archiveSize)
|
||||
return data ? { from, data } : null
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
/**
|
||||
* Separating equal-sized obfuscated volumes by posting order, independent of AdonisJS.
|
||||
*/
|
||||
export { splitInstances, splitByRank, splitVerified, verifyVolume, type SplitArticle, type SplitOptions } from './instances.js'
|
||||
@@ -1,178 +0,0 @@
|
||||
import { combineCrc32 } from '../yenc/decode.js'
|
||||
|
||||
export interface SplitArticle {
|
||||
number: number
|
||||
messageId: string
|
||||
bytes: number
|
||||
/** Article number in the group; posting order. */
|
||||
articleNumber: number | null
|
||||
/** yEnc part CRC and decoded part length, for verification. */
|
||||
pcrc32?: string | null
|
||||
partSize?: number | null
|
||||
/** Whole-file CRC32, usually only in the last part. */
|
||||
fileCrc32?: string | null
|
||||
}
|
||||
|
||||
export interface SplitOptions {
|
||||
/** Total parts per volume; a volume that has them all takes no more articles. */
|
||||
totalParts: number
|
||||
/**
|
||||
* Maximum distance in article numbers between consecutive articles of one
|
||||
* volume. Separates different posts that happen to share a size.
|
||||
*/
|
||||
window?: number
|
||||
/**
|
||||
* How far (in posting positions) an article may be from where its volume
|
||||
* predicts it; covers out-of-order finishing across connections.
|
||||
*/
|
||||
tolerance?: number
|
||||
}
|
||||
|
||||
interface Volume {
|
||||
articles: SplitArticle[]
|
||||
/** Posting rank of each part number present. */
|
||||
ranks: Map<number, number>
|
||||
lastArticle: number
|
||||
opened: number
|
||||
}
|
||||
|
||||
/** ~1 minute of posting in a busy group. */
|
||||
const DEFAULT_WINDOW = 20_000
|
||||
|
||||
/**
|
||||
* Separates the articles of several equal-sized files that share one
|
||||
* identity (obfuscated volumes with random names, possibly from different
|
||||
* posts) into one list per file. See CONCEPT.md (D20).
|
||||
*
|
||||
* Posting tools upload a queue (volume 1 parts 1..n, volume 2 parts 1..n,
|
||||
* ...) over several connections, so within a volume part k+1 lands about one
|
||||
* position after part k in posting order, give or take the connection count.
|
||||
* Each article, in posting order, goes to the eligible volume (lacks this
|
||||
* part, not complete, active within `window` article numbers) whose
|
||||
* predicted position for it is closest: taken from that volume's article with
|
||||
* the nearest part number, which keeps the prediction local and unaffected
|
||||
* by missing articles elsewhere. A deviation above the tolerance starts a new
|
||||
* volume. The result is recomputed from all articles, so it improves as late
|
||||
* articles arrive.
|
||||
*/
|
||||
export function splitInstances(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
|
||||
const window = options.window ?? DEFAULT_WINDOW
|
||||
const tolerance = options.tolerance ?? Math.max(16, Math.ceil(options.totalParts * 0.1))
|
||||
const seen = new Set<string>()
|
||||
const ordered = articles
|
||||
.filter((a) => (seen.has(a.messageId) ? false : (seen.add(a.messageId), true)))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
|
||||
a.messageId.localeCompare(b.messageId)
|
||||
)
|
||||
|
||||
const volumes: Volume[] = []
|
||||
ordered.forEach((article, rank) => {
|
||||
const position = article.articleNumber ?? Number.MAX_SAFE_INTEGER
|
||||
let best: Volume | undefined
|
||||
let bestScore: [number, number] | undefined
|
||||
for (const volume of volumes) {
|
||||
if (volume.ranks.has(article.number) || volume.ranks.size >= options.totalParts) continue
|
||||
if (position - volume.lastArticle > window) continue
|
||||
const deviation = Math.abs(rank - predictedRank(volume, article.number))
|
||||
if (deviation > tolerance) continue
|
||||
const score: [number, number] = [deviation, volume.opened]
|
||||
if (!bestScore || score[0] < bestScore[0] || (score[0] === bestScore[0] && score[1] < bestScore[1])) {
|
||||
best = volume
|
||||
bestScore = score
|
||||
}
|
||||
}
|
||||
if (!best) {
|
||||
best = { articles: [], ranks: new Map(), lastArticle: position, opened: volumes.length }
|
||||
volumes.push(best)
|
||||
}
|
||||
best.articles.push(article)
|
||||
best.ranks.set(article.number, rank)
|
||||
best.lastArticle = Math.max(best.lastArticle, position)
|
||||
})
|
||||
|
||||
return volumes.map((v) => v.articles.sort((a, b) => a.number - b.number))
|
||||
}
|
||||
|
||||
/** Where part `number` should be in posting order, judged from the volume's nearest part. */
|
||||
function predictedRank(volume: Volume, number: number): number {
|
||||
let anchorPart = 0
|
||||
let anchorRank = 0
|
||||
let nearest = Number.MAX_SAFE_INTEGER
|
||||
for (const [part, rank] of volume.ranks) {
|
||||
const distance = Math.abs(part - number)
|
||||
if (distance < nearest) {
|
||||
nearest = distance
|
||||
anchorPart = part
|
||||
anchorRank = rank
|
||||
}
|
||||
}
|
||||
return anchorRank + (number - anchorPart)
|
||||
}
|
||||
|
||||
/**
|
||||
* Simpler split: for every part number, the i-th article in posting order
|
||||
* belongs to volume i. Right for a single, complete post; used as a second
|
||||
* proposal for verification.
|
||||
*/
|
||||
export function splitByRank(articles: SplitArticle[]): SplitArticle[][] {
|
||||
const byNumber = new Map<number, SplitArticle[]>()
|
||||
const seen = new Set<string>()
|
||||
for (const article of articles) {
|
||||
if (seen.has(article.messageId)) continue
|
||||
seen.add(article.messageId)
|
||||
byNumber.set(article.number, [...(byNumber.get(article.number) ?? []), article])
|
||||
}
|
||||
const volumes: SplitArticle[][] = []
|
||||
for (const group of byNumber.values()) {
|
||||
group.sort(
|
||||
(a, b) =>
|
||||
(a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
|
||||
a.messageId.localeCompare(b.messageId)
|
||||
)
|
||||
group.forEach((article, i) => (volumes[i] ??= []).push(article))
|
||||
}
|
||||
return volumes.map((v) => v.sort((a, b) => a.number - b.number))
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a volume has every part and its combined part CRCs equal the
|
||||
* whole-file CRC32 its last part announces: then it is exactly one file.
|
||||
*/
|
||||
export function verifyVolume(volume: SplitArticle[], totalParts: number): boolean {
|
||||
if (volume.length !== totalParts) return false
|
||||
const sorted = [...volume].sort((a, b) => a.number - b.number)
|
||||
if (sorted.some((a, i) => a.number !== i + 1 || !a.pcrc32 || !a.partSize)) return false
|
||||
const expected = sorted[sorted.length - 1].fileCrc32
|
||||
if (!expected) return false
|
||||
const combined = combineCrc32(sorted.map((a) => ({ crc: a.pcrc32!, length: a.partSize! })))
|
||||
return combined === expected.toLowerCase().padStart(8, '0')
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits and keeps only volumes that verify (verifyVolume). Proposals from
|
||||
* both methods are tried; articles of verified volumes are taken out and the
|
||||
* rest is tried again, until nothing new verifies. Unverified articles are
|
||||
* left out: guessing wrong would produce a corrupt file.
|
||||
*/
|
||||
export function splitVerified(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
|
||||
const verified: SplitArticle[][] = []
|
||||
let remaining = articles
|
||||
for (let round = 0; round < 4 && remaining.length >= options.totalParts; round++) {
|
||||
const found: SplitArticle[][] = []
|
||||
const used = new Set<string>()
|
||||
for (const propose of [(a: SplitArticle[]) => splitInstances(a, options), splitByRank]) {
|
||||
for (const volume of propose(remaining.filter((a) => !used.has(a.messageId)))) {
|
||||
if (verifyVolume(volume, options.totalParts)) {
|
||||
found.push(volume)
|
||||
for (const article of volume) used.add(article.messageId)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found.length === 0) break
|
||||
verified.push(...found)
|
||||
remaining = remaining.filter((a) => !used.has(a.messageId))
|
||||
}
|
||||
return verified
|
||||
}
|
||||
@@ -1,333 +0,0 @@
|
||||
/**
|
||||
* Spotnet's category tree, from Spotweb's SpotCategories (English names).
|
||||
* A spot has a head category (0–3) and subcategories per type letter:
|
||||
* `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre,
|
||||
* `d` genre, `z` kind (movie, series, book, …).
|
||||
*/
|
||||
export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications']
|
||||
|
||||
/** What each type letter means per head category. */
|
||||
export const SPOT_SUBCAT_TYPES: Record<string, string>[] = [
|
||||
{ a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' },
|
||||
{ a: 'Platform', b: 'Genre', z: 'Type' },
|
||||
]
|
||||
|
||||
const NAMES: Record<string, Record<number, string>>[] = [
|
||||
{
|
||||
a: {
|
||||
0: 'DivX',
|
||||
1: 'WMV',
|
||||
2: 'MPG',
|
||||
3: 'DVD5',
|
||||
4: 'HD other',
|
||||
5: 'ePub',
|
||||
6: 'Blu-ray',
|
||||
7: 'HD-DVD',
|
||||
8: 'WMVHD',
|
||||
9: 'x264',
|
||||
10: 'DVD9',
|
||||
11: 'PDF',
|
||||
12: 'Bitmap',
|
||||
13: 'Vector',
|
||||
14: '3D',
|
||||
15: 'UHD',
|
||||
},
|
||||
b: {
|
||||
0: 'CAM',
|
||||
1: '(S)VCD',
|
||||
2: 'Promo',
|
||||
3: 'Retail',
|
||||
4: 'TV',
|
||||
6: 'Satellite',
|
||||
7: 'R5',
|
||||
8: 'Telecine',
|
||||
9: 'Telesync',
|
||||
10: 'Scan',
|
||||
11: 'WEB-DL',
|
||||
12: 'WEBRip',
|
||||
13: 'HDRip',
|
||||
},
|
||||
c: {
|
||||
0: 'No subtitles',
|
||||
1: 'Dutch subtitles (external)',
|
||||
2: 'Dutch subtitles (builtin)',
|
||||
3: 'English subtitles (external)',
|
||||
4: 'English subtitles (builtin)',
|
||||
6: 'Dutch subtitles (available)',
|
||||
7: 'English subtitles (available)',
|
||||
10: 'English audio/written',
|
||||
11: 'Dutch audio/written',
|
||||
12: 'German audio/written',
|
||||
13: 'French audio/written',
|
||||
14: 'Spanish audio/written',
|
||||
15: 'Asian audio/written',
|
||||
},
|
||||
d: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Animation',
|
||||
3: 'Cabaret',
|
||||
4: 'Comedy',
|
||||
5: 'Crime',
|
||||
6: 'Documentary',
|
||||
7: 'Drama',
|
||||
8: 'Family',
|
||||
9: 'Fantasy',
|
||||
10: 'Arthouse',
|
||||
11: 'Television',
|
||||
12: 'Horror',
|
||||
13: 'Music',
|
||||
14: 'Musical',
|
||||
15: 'Mystery',
|
||||
16: 'Romance',
|
||||
17: 'Science Fiction',
|
||||
18: 'Sport',
|
||||
19: 'Short movie',
|
||||
20: 'Thriller',
|
||||
21: 'War',
|
||||
22: 'Western',
|
||||
23: 'Erotica (hetero)',
|
||||
24: 'Erotica (gay male)',
|
||||
25: 'Erotica (gay female)',
|
||||
26: 'Erotica (bi)',
|
||||
28: 'Asian',
|
||||
29: 'Anime',
|
||||
30: 'Cover',
|
||||
31: 'Comicbook',
|
||||
32: 'Cartoons',
|
||||
33: 'Youth',
|
||||
34: 'Business',
|
||||
35: 'Computer',
|
||||
36: 'Hobby',
|
||||
37: 'Cooking',
|
||||
38: 'Handwork',
|
||||
39: 'Craftwork',
|
||||
40: 'Health',
|
||||
41: 'History',
|
||||
42: 'Psychology',
|
||||
43: 'Newspaper',
|
||||
44: 'Magazine',
|
||||
45: 'Science',
|
||||
46: 'Female',
|
||||
47: 'Religion',
|
||||
48: 'Roman',
|
||||
49: 'Biography',
|
||||
50: 'Detective',
|
||||
51: 'Animals',
|
||||
52: 'Humor',
|
||||
53: 'Travel',
|
||||
54: 'True story',
|
||||
55: 'Non-fiction',
|
||||
56: 'Politics',
|
||||
57: 'Poetry',
|
||||
58: 'Fairy tale',
|
||||
59: 'Technical',
|
||||
60: 'Art',
|
||||
72: 'Bi',
|
||||
73: 'Lesbian',
|
||||
74: 'Homo',
|
||||
75: 'Hetero',
|
||||
76: 'Amature',
|
||||
77: 'Group',
|
||||
78: 'POV',
|
||||
79: 'Solo',
|
||||
80: 'Young',
|
||||
81: 'Soft',
|
||||
82: 'Fetish',
|
||||
83: 'Old',
|
||||
84: 'Fat',
|
||||
85: 'SM',
|
||||
86: 'Rough',
|
||||
87: 'Dark',
|
||||
88: 'Hentai',
|
||||
89: 'Outside',
|
||||
},
|
||||
z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'MP3',
|
||||
1: 'WMA',
|
||||
2: 'WAV',
|
||||
3: 'OGG',
|
||||
4: 'EAC',
|
||||
5: 'DTS',
|
||||
6: 'AAC',
|
||||
7: 'APE',
|
||||
8: 'FLAC',
|
||||
},
|
||||
b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' },
|
||||
c: {
|
||||
0: 'Variable',
|
||||
1: '< 96kbit',
|
||||
2: '96kbit',
|
||||
3: '128kbit',
|
||||
4: '160kbit',
|
||||
5: '192kbit',
|
||||
6: '256kbit',
|
||||
7: '320kbit',
|
||||
8: 'Lossless',
|
||||
9: 'Other',
|
||||
},
|
||||
d: {
|
||||
0: 'Blues',
|
||||
1: 'Compilation',
|
||||
2: 'Cabaret',
|
||||
3: 'Dance',
|
||||
4: 'Diverse',
|
||||
5: 'Hardcore',
|
||||
6: 'World',
|
||||
7: 'Jazz',
|
||||
8: 'Youth',
|
||||
9: 'Classical',
|
||||
10: 'Kleinkunst',
|
||||
11: 'Dutch',
|
||||
12: 'New Age',
|
||||
13: 'Pop',
|
||||
14: 'RnB',
|
||||
15: 'Hiphop',
|
||||
16: 'Reggae',
|
||||
17: 'Religious',
|
||||
18: 'Rock',
|
||||
19: 'Soundtracks',
|
||||
20: 'Other',
|
||||
21: 'Hardstyle',
|
||||
22: 'Asian',
|
||||
23: 'Disco',
|
||||
24: 'Classics',
|
||||
25: 'Metal',
|
||||
26: 'Country',
|
||||
27: 'Dubstep',
|
||||
28: 'Nederhop',
|
||||
29: 'DnB',
|
||||
30: 'Electro',
|
||||
31: 'Folk',
|
||||
32: 'Soul',
|
||||
33: 'Trance',
|
||||
34: 'Balkan',
|
||||
35: 'Techno',
|
||||
36: 'Ambient',
|
||||
37: 'Latin',
|
||||
38: 'Live',
|
||||
},
|
||||
z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' },
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'Playstation',
|
||||
4: 'Playstation 2',
|
||||
5: 'PSP',
|
||||
6: 'Xbox',
|
||||
7: 'Xbox 360',
|
||||
8: 'Gameboy Advance',
|
||||
9: 'Gamecube',
|
||||
10: 'Nintendo DS',
|
||||
11: 'Nintento Wii',
|
||||
12: 'Playstation 3',
|
||||
13: 'Windows Phone',
|
||||
14: 'iOS',
|
||||
15: 'Android',
|
||||
16: 'Nintendo 3DS',
|
||||
17: 'Playstation 4',
|
||||
18: 'XBox 1',
|
||||
},
|
||||
b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' },
|
||||
c: {
|
||||
0: 'Action',
|
||||
1: 'Adventure',
|
||||
2: 'Strategy',
|
||||
3: 'Roleplaying',
|
||||
4: 'Simulation',
|
||||
5: 'Race',
|
||||
6: 'Flying',
|
||||
7: 'Shooter',
|
||||
8: 'Platform',
|
||||
9: 'Sport',
|
||||
10: 'Child/youth',
|
||||
11: 'Puzzle',
|
||||
12: 'Other',
|
||||
13: 'Boardgame',
|
||||
14: 'Cards',
|
||||
15: 'Education',
|
||||
16: 'Music',
|
||||
17: 'Family',
|
||||
},
|
||||
},
|
||||
{
|
||||
a: {
|
||||
0: 'Windows',
|
||||
1: 'Macintosh',
|
||||
2: 'Linux',
|
||||
3: 'OS/2',
|
||||
4: 'Windows Phone',
|
||||
5: 'Navigation systems',
|
||||
6: 'iOS',
|
||||
7: 'Android',
|
||||
},
|
||||
b: {
|
||||
0: 'Audio',
|
||||
1: 'Video',
|
||||
2: 'Graphics',
|
||||
3: 'CD/DVD Tools',
|
||||
4: 'Media players',
|
||||
5: 'Rippers & Encoders',
|
||||
6: 'Plugins',
|
||||
7: 'Database tools',
|
||||
8: 'Email software',
|
||||
9: 'Photo',
|
||||
10: 'Screensavers',
|
||||
11: 'Skin software',
|
||||
12: 'Drivers',
|
||||
13: 'Browsers',
|
||||
14: 'Download managers',
|
||||
15: 'Download',
|
||||
16: 'Usenet software',
|
||||
17: 'RSS Readers',
|
||||
18: 'FTP software',
|
||||
19: 'Firewalls',
|
||||
20: 'Antivirus software',
|
||||
21: 'Antispyware software',
|
||||
22: 'Optimization software',
|
||||
23: 'Security software',
|
||||
24: 'System software',
|
||||
25: 'Other',
|
||||
26: 'Educational',
|
||||
27: 'Office',
|
||||
28: 'Internet',
|
||||
29: 'Communication',
|
||||
30: 'Development',
|
||||
31: 'Spotnet',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
export interface SpotCategoryLabels {
|
||||
head: string | null
|
||||
/** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */
|
||||
subcats: { type: string; values: string[] }[]
|
||||
}
|
||||
|
||||
/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */
|
||||
export function spotCategoryLabels(
|
||||
category: number | null,
|
||||
subcats: Record<string, number[]> | null
|
||||
): SpotCategoryLabels {
|
||||
if (category === null || !(category in NAMES)) return { head: null, subcats: [] }
|
||||
const labels: SpotCategoryLabels['subcats'] = []
|
||||
for (const letter of ['z', 'a', 'b', 'c', 'd']) {
|
||||
const values = [
|
||||
...new Set(
|
||||
(subcats?.[letter] ?? [])
|
||||
.map((n) => NAMES[category][letter]?.[n])
|
||||
.filter((v): v is string => Boolean(v))
|
||||
),
|
||||
]
|
||||
if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values })
|
||||
}
|
||||
return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels }
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
import { createHash, createPublicKey, verify } from 'node:crypto'
|
||||
import { crc32 } from 'node:zlib'
|
||||
|
||||
/**
|
||||
* Spotnet spot headers (free.pt), after Spotweb's Services_Format_Parsing::parseHeader.
|
||||
*
|
||||
* From: Nick <[PUBKEY[.USERSIGNATURE]]@[CAT][KEYID][SUBCATS].[SIZE].[x].[DATE].[..].[SIGNATURE]>
|
||||
* Subject: Title | Tag
|
||||
*/
|
||||
export interface SpotHeader {
|
||||
messageId: string
|
||||
poster: string
|
||||
title: string
|
||||
tag: string
|
||||
/** 0 image/video, 1 sound, 2 games, 3 applications. */
|
||||
category: number
|
||||
keyId: number
|
||||
/** e.g. { a: [9], b: [3], c: [7, 10], d: [17], z: [0] } */
|
||||
subcats: Record<string, number[]>
|
||||
size: number
|
||||
/** Unix seconds. */
|
||||
stamp: number
|
||||
/** Everything after the @. */
|
||||
header: string
|
||||
headerSign: string
|
||||
selfSignedPubKey: string
|
||||
userSignature?: string
|
||||
}
|
||||
|
||||
/** Spotnet's base64 variant: `-s` for `/`, `-p` for `+`, padding left out. */
|
||||
export function spotUnprepareBase64(input: string): string {
|
||||
const padded = input.length % 4 ? input + '='.repeat(4 - (input.length % 4)) : input
|
||||
return padded.replace(/-s/g, '/').replace(/-p/g, '+')
|
||||
}
|
||||
|
||||
/** Parses the From/Subject of a spot. Null when it isn't one. */
|
||||
export function parseSpotHeader(from: string, subject: string, messageId: string, date?: string): SpotHeader | null {
|
||||
const lt = from.indexOf('<')
|
||||
if (lt === -1) return null
|
||||
const address = from.slice(lt + 1, from.endsWith('>') ? -1 : undefined)
|
||||
const at = address.lastIndexOf('@')
|
||||
if (at === -1) return null
|
||||
const header = address.slice(at + 1)
|
||||
const [pubKey, userSignature] = address.slice(0, at).split('.')
|
||||
|
||||
const fields = header.split('.')
|
||||
if (fields.length < 6) return null
|
||||
const category = Number.parseInt(fields[0][0], 10) - 1
|
||||
const keyId = Number.parseInt(fields[0][1], 10)
|
||||
if (!(category >= 0) || !(keyId >= 0)) return null
|
||||
|
||||
// a9b4c0d5 (FTD) or a09b04c00d05 (newer): a letter, then digits.
|
||||
const subcats: Record<string, number[]> = {}
|
||||
for (const [, letter, digits] of fields[0].slice(2).toLowerCase().matchAll(/([a-z])(\d+)/g)) {
|
||||
if ('abcdz'.includes(letter)) (subcats[letter] ??= []).push(Number.parseInt(digits, 10))
|
||||
}
|
||||
|
||||
// Newer keys: "Title | Tag". Key 1 (oldest): the tag is the last field.
|
||||
const parts = subject.split('|')
|
||||
let title: string
|
||||
let tag = ''
|
||||
if (keyId !== 1) {
|
||||
title = parts[0].trim()
|
||||
tag = parts.length > 1 ? parts[1].trim() : ''
|
||||
} else {
|
||||
tag = parts.length > 1 ? parts[parts.length - 1].trim() : subject.trim()
|
||||
title = parts.length > 1 ? parts.slice(0, -2).join('|').trim() : ''
|
||||
}
|
||||
|
||||
const parsedDate = date ? Date.parse(date) : Number.NaN
|
||||
return {
|
||||
messageId: messageId.replace(/^<|>$/g, ''),
|
||||
poster: from.slice(0, lt).trim(),
|
||||
title,
|
||||
tag,
|
||||
category,
|
||||
keyId,
|
||||
subcats,
|
||||
size: Number.parseInt(fields[1], 10) || 0,
|
||||
stamp: Number.isNaN(parsedDate) ? Number.parseInt(fields[3], 10) || 0 : Math.floor(parsedDate / 1000),
|
||||
header,
|
||||
headerSign: fields[fields.length - 1],
|
||||
selfSignedPubKey: spotUnprepareBase64(pubKey ?? ''),
|
||||
userSignature: userSignature ? spotUnprepareBase64(userSignature) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/** Spotnet's own signing keys (Spotweb settings `rsa_keys`), base64 modulus. */
|
||||
export const SPOTNET_KEYS: Record<number, string> = {
|
||||
2: 'ys8WSlqonQMWT8ubG0tAA2Q07P36E+CJmb875wSR1XH7IFhEi0CCwlUzNqBFhC+P',
|
||||
3: 'uiyChPV23eguLAJNttC/o0nAsxXgdjtvUvidV2JL+hjNzc4Tc/PPo2JdYvsqUsat',
|
||||
4: '1k6RNDVD6yBYWR6kHmwzmSud7JkNV4SMigBrs+jFgOK5Ldzwl17mKXJhl+su/GR9',
|
||||
}
|
||||
|
||||
const base64url = (b64: string) => Buffer.from(b64, 'base64').toString('base64url')
|
||||
|
||||
/** RSA PKCS#1 v1.5 with SHA-1, as Spotweb's checkRsaSignature (exponent AQAB). */
|
||||
export function checkRsaSignature(message: Buffer | string, signatureB64: string, modulusB64: string): boolean {
|
||||
try {
|
||||
const key = createPublicKey({ key: { kty: 'RSA', n: base64url(modulusB64), e: 'AQAB' }, format: 'jwk' })
|
||||
const data = typeof message === 'string' ? toBytes(message) : message
|
||||
return verify('sha1', data, key, Buffer.from(signatureB64, 'base64'))
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Spots are signed over their ISO-8859-1 bytes. */
|
||||
function toBytes(text: string): Buffer {
|
||||
return /^[\u0000-ÿ]*$/.test(text) ? Buffer.from(text, 'latin1') : Buffer.from(text, 'utf8')
|
||||
}
|
||||
|
||||
/** SPOTSIGN_V2 and self-signed messages carry a hashcash: sha1('<msgid>') starts with 0000. */
|
||||
function hashcash(messageId: string): boolean {
|
||||
return createHash('sha1').update(`<${messageId}>`).digest('hex').startsWith('0000')
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies a spot header like Spotweb does: key 7 (self-signed) needs a
|
||||
* hashcash and the poster's key, keys 2–4 Spotnet's keys; spots that also
|
||||
* carry a user signature must have it match the message id.
|
||||
*/
|
||||
export function verifySpotHeader(spot: SpotHeader, keys: Record<number, string> = SPOTNET_KEYS): boolean {
|
||||
if (!spot.title || !spot.poster) return false
|
||||
// Old key 1 spots from before 2011 weren't signed.
|
||||
if (spot.keyId === 1 && spot.stamp <= 1293870080) return true
|
||||
if (!spot.headerSign) return false
|
||||
|
||||
const signature = spotUnprepareBase64(spot.headerSign)
|
||||
const toCheck = spot.title + spot.header.slice(0, spot.header.length - spot.headerSign.length - 1) + spot.poster
|
||||
let verified: boolean
|
||||
if (spot.keyId === 7 || (spot.keyId === 2 && spot.selfSignedPubKey.length > 50)) {
|
||||
// Self-signed (and key-2 "dispose" messages): the hashcash is the gate, as in Spotweb.
|
||||
verified = hashcash(spot.messageId)
|
||||
} else {
|
||||
verified = keys[spot.keyId] !== undefined && checkRsaSignature(toCheck, signature, keys[spot.keyId])
|
||||
}
|
||||
if (verified && spot.userSignature && spot.selfSignedPubKey) {
|
||||
verified = checkRsaSignature(`<${spot.messageId}>`, spot.userSignature, spot.selfSignedPubKey)
|
||||
}
|
||||
return verified
|
||||
}
|
||||
|
||||
/** Spotweb's spotter id: base64 of the CRC32 (little-endian) of the poster's public key. */
|
||||
export function spotterId(pubKeyB64: string): string {
|
||||
if (!pubKeyB64) return ''
|
||||
const crc = Buffer.alloc(4)
|
||||
crc.writeUInt32LE(crc32(Buffer.from(pubKeyB64, 'base64')) >>> 0)
|
||||
return crc.toString('base64').replace(/[/+=]/g, '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Moderation: key-2 spots titled "delete|dispose|remove <message-id>".
|
||||
* From Spotnet's key they are moderator actions; with the poster's own key
|
||||
* (a spotter id) a personal dispose of their own spot.
|
||||
*/
|
||||
export function moderationTarget(spot: Pick<SpotHeader, 'keyId' | 'title'>): string | null {
|
||||
if (spot.keyId !== 2) return null
|
||||
const [command, target] = spot.title.split(' ')
|
||||
if (!['delete', 'dispose', 'remove'].includes(command?.toLowerCase()) || !target) return null
|
||||
return target.replace(/^<|>$/g, '')
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
/**
|
||||
* Spotnet (free.pt) spots, independent of AdonisJS: header parsing and
|
||||
* signature checks after Spotweb, the spot XML and its compressed NZB.
|
||||
* See CONCEPT.md (D29).
|
||||
*/
|
||||
export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
|
||||
export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, decodeSpotText, type SpotXml, type SpotImage } from './xml.js'
|
||||
export { spotCategoryLabels, SPOT_HEAD_CATEGORIES, type SpotCategoryLabels } from './categories.js'
|
||||
export { passwordFromSpot } from './password.js'
|
||||
@@ -1,83 +0,0 @@
|
||||
/**
|
||||
* Archive passwords that spotters put in the title or description instead of
|
||||
* the NZB's `<meta type="password">`:
|
||||
* `Title{{secret}}` (the newznab/SABnzbd convention),
|
||||
* `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`,
|
||||
* a few words between keyword and separator (`Het wachtwoord voor de nzb is: dw0067`,
|
||||
* `ww in nzb: X`, `Wachtwoord inbegrepen: wysiwyg`),
|
||||
* the value on the next line (`WW:` / `tijdloos`),
|
||||
* `wachtwoord is arn2`, `… voorzien van een wachtwoord. Deze is: X`,
|
||||
* `WW FC0401` (abbreviation without separator: with a digit in it, or as the whole line).
|
||||
* The value must be the rest of its line and a single token, so sentences
|
||||
* such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or
|
||||
* "met een password beschermd" don't count (LOG.md 2026-10-08 (10)).
|
||||
*/
|
||||
const KEYWORD = String.raw`(?:wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw)(?![a-z])`
|
||||
const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}(?:\s+[a-zà-ÿ]+){0,3}?\s*(?:is\s*)?[:=]\s*(.*)$`, 'i')
|
||||
const IS_VALUE = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}\s+is\s+(\S+)\s*$`, 'i')
|
||||
/** "Deze is voorzien van een wachtwoord. Deze is: X": the value in the next sentence. */
|
||||
const NEXT_SENTENCE = /(?:^|[.!?]\s*)(?:deze|dit|het|die|this|it)\s+is\s*[:=]\s*(\S+)\s*$/i
|
||||
const MENTIONS = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}`, 'i')
|
||||
const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i
|
||||
/** "WW carrie" as a line of its own; "WW II" is the war. */
|
||||
const WHOLE_LINE_ABBREVIATION = /^(?:ww|pw)\s+(\S+)$/i
|
||||
const NOT_PASSWORD_TOKEN = /^(?:i{1,3}|iv|v|[12])$/i
|
||||
const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i
|
||||
const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i
|
||||
|
||||
/** Lines of a spot description: UBB/HTML line breaks split, other tags dropped. */
|
||||
function lines(description: string): string[] {
|
||||
return description
|
||||
.replace(/\[br\]|<br\s*\/?>/gi, '\n')
|
||||
.replace(/\[\/?[a-z]+(?:=[^\]]*)?\]|<\/?[a-z][^>]*>/gi, '')
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
}
|
||||
|
||||
function clean(value: string): string | null {
|
||||
const token = value
|
||||
.trim()
|
||||
// "Wachtwoord : yakowyke !!"
|
||||
.replace(/^[!*~#]+\s*|\s*[!*~#]+$/g, '')
|
||||
.replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '')
|
||||
if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null
|
||||
return token
|
||||
}
|
||||
|
||||
export function passwordFromSpot(title: string, description?: string | null): string | null {
|
||||
const braces = /\{\{([^{}]{1,64})\}\}/.exec(title)
|
||||
if (braces && clean(braces[1])) return clean(braces[1])
|
||||
const all = lines(description ?? '')
|
||||
for (const [i, line] of all.entries()) {
|
||||
if (!line || NO_PASSWORD.test(line)) continue
|
||||
const separated = WITH_SEPARATOR.exec(line)
|
||||
if (separated) {
|
||||
const value = separated[1].trim() ? clean(separated[1]) : nextLineValue(all, i)
|
||||
if (value) return value
|
||||
continue
|
||||
}
|
||||
const is = IS_VALUE.exec(line)
|
||||
if (is) {
|
||||
const value = clean(is[1])
|
||||
if (value) return value
|
||||
}
|
||||
const sentence = MENTIONS.test(line) ? NEXT_SENTENCE.exec(line) : null
|
||||
if (sentence) {
|
||||
const value = clean(sentence[1])
|
||||
if (value) return value
|
||||
}
|
||||
const whole = WHOLE_LINE_ABBREVIATION.exec(line)
|
||||
const abbreviated = whole ?? ABBREVIATION.exec(line)
|
||||
if (abbreviated && (whole ? !NOT_PASSWORD_TOKEN.test(abbreviated[1]) : /\d/.test(abbreviated[1]))) {
|
||||
const value = clean(abbreviated[1])
|
||||
if (value) return value
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** `WW:` with the value alone on the next non-empty line. */
|
||||
function nextLineValue(all: string[], i: number): string | null {
|
||||
const next = all.slice(i + 1).find((line) => line !== '')
|
||||
return next ? clean(next) : null
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
import { inflateRawSync } from 'node:zlib'
|
||||
|
||||
export interface SpotXml {
|
||||
title: string
|
||||
category: number | null
|
||||
subcats: string[]
|
||||
size: number
|
||||
website: string
|
||||
/** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
|
||||
description: string
|
||||
tag: string
|
||||
/** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
|
||||
image: SpotImage | null
|
||||
/** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
|
||||
nzbSegments: string[]
|
||||
created: number | null
|
||||
}
|
||||
|
||||
export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
|
||||
|
||||
/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
|
||||
function text(value: string | undefined): string {
|
||||
if (value === undefined) return ''
|
||||
const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
|
||||
return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
|
||||
if (entity[0] === '#') {
|
||||
const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
|
||||
}
|
||||
return ENTITIES[entity.toLowerCase()] ?? match
|
||||
})
|
||||
}
|
||||
|
||||
const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, 'i').exec(xml)?.[1]
|
||||
|
||||
/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */
|
||||
export function parseSpotXml(xml: string): SpotXml | null {
|
||||
const posting = element(xml, 'Posting')
|
||||
if (!posting) return null
|
||||
const categoryXml = element(posting, 'Category') ?? ''
|
||||
const key = Number.parseInt(element(posting, 'Key') ?? '', 10)
|
||||
const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
|
||||
const nzb = element(posting, 'NZB') ?? ''
|
||||
return {
|
||||
description: text(element(posting, 'Description')).trim(),
|
||||
tag: text(element(posting, 'Tag')).trim(),
|
||||
image: parseImage(posting),
|
||||
title: text(element(posting, 'Title')).trim(),
|
||||
// Newer keys number categories from 1.
|
||||
category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
|
||||
subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()),
|
||||
size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0,
|
||||
website: text(element(posting, 'Website')).trim(),
|
||||
nzbSegments: [...nzb.matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean),
|
||||
created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null,
|
||||
}
|
||||
}
|
||||
|
||||
function parseImage(posting: string): SpotImage | null {
|
||||
const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
|
||||
if (!match) return null
|
||||
const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
|
||||
if (segments.length > 0) {
|
||||
const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
|
||||
return { segments, width: size('Width'), height: size('Height') }
|
||||
}
|
||||
const url = text(match[2]).trim()
|
||||
return /^https?:\/\//i.test(url) ? { url } : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
|
||||
* the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
|
||||
*/
|
||||
export function decodeSpotText(latin1: string): string {
|
||||
const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
|
||||
return utf8.includes('\uFFFD') ? latin1 : utf8
|
||||
}
|
||||
|
||||
/**
|
||||
* Header lines of a raw HEAD response; X-XML (split over many lines) joined.
|
||||
* Folded headers are unfolded: old (key 1) spots put the From address on a
|
||||
* continuation line.
|
||||
*/
|
||||
export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
|
||||
const headers: Record<string, string> = {}
|
||||
let xml = ''
|
||||
let last: string | null = null
|
||||
let lastIsFirst = false
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
if (/^[ \t]/.test(line) && last) {
|
||||
if (last === 'x-xml') xml += line
|
||||
else if (lastIsFirst) headers[last] += line
|
||||
continue
|
||||
}
|
||||
const colon = line.indexOf(':')
|
||||
if (colon <= 0) {
|
||||
last = null
|
||||
continue
|
||||
}
|
||||
const name = line.slice(0, colon).toLowerCase()
|
||||
const value = line.slice(colon + 1).replace(/^ /, '')
|
||||
last = name
|
||||
lastIsFirst = !(name in headers)
|
||||
if (name === 'x-xml') xml += value
|
||||
else headers[name] ??= value
|
||||
}
|
||||
return { headers, xml }
|
||||
}
|
||||
|
||||
/**
|
||||
* Spot binaries (the NZB, images): article bodies joined without line
|
||||
* breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw
|
||||
* deflate (PHP gzinflate).
|
||||
*/
|
||||
export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer {
|
||||
const joined = bodies
|
||||
.map((body) =>
|
||||
body
|
||||
.toString('latin1')
|
||||
.split('\r\n')
|
||||
// NNTP dot-stuffing.
|
||||
.map((line) => (line.startsWith('..') ? line.slice(1) : line))
|
||||
.join('')
|
||||
)
|
||||
.join('')
|
||||
const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1')
|
||||
return compressed ? inflateRawSync(unescaped) : unescaped
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
|
||||
/**
|
||||
* Identity of one uploaded file: every segment of it has the same poster, the
|
||||
* same subject apart from the segment counter, and the same segment count.
|
||||
* The group is deliberately left out so crossposts resolve to one file.
|
||||
* See CONCEPT.md (D12).
|
||||
*/
|
||||
export function fileKey(poster: string, base: string, totalSegments: number): string {
|
||||
return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex')
|
||||
}
|
||||
|
||||
/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */
|
||||
export function looksLikeFilename(name: string): boolean {
|
||||
return /\.[A-Za-z0-9]{1,5}$/.test(name) && !/^[0-9a-f]{32,}$/i.test(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of a file known only from its yEnc header (obfuscated posts).
|
||||
* The poster is left out: obfuscating posters randomise it per article.
|
||||
* - A real filename is stable across the articles of a file, and needed to
|
||||
* tell apart the equal-sized volumes of one archive.
|
||||
* - A random, extensionless name can change per article too; the exact
|
||||
* byte size and part count are what stays the same.
|
||||
* Collisions are caught by the conflict check in IngestService.
|
||||
*/
|
||||
export function weakFileKey(name: string, size: number, totalParts: number): string {
|
||||
const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape'
|
||||
return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of a file by its counters, for uploads that give every article
|
||||
* its own random subject (CONCEPT.md D36): poster, file counter `[i/M]`,
|
||||
* part count and posting day stay the same across the articles of one file.
|
||||
* The day keeps different uploads with the same shape apart; collisions that
|
||||
* remain are caught by the conflict check (these files are weak identities).
|
||||
* Null when the subject has no file counter.
|
||||
*/
|
||||
export function counterFileKey(segment: { poster: string; subject: string; totalParts: number; postedAt?: Date }): string | null {
|
||||
const counter = /\[(\d+)\/(\d+)\]/.exec(segment.subject)
|
||||
if (!counter || segment.totalParts < 2) return null
|
||||
const day = segment.postedAt && !Number.isNaN(segment.postedAt.getTime()) ? segment.postedAt.toISOString().slice(0, 10) : ''
|
||||
return createHash('sha1')
|
||||
.update(`counter\n${segment.poster}\n${Number(counter[1])}/${Number(counter[2])}\n${segment.totalParts}\n${day}`)
|
||||
.digest('hex')
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
/**
|
||||
* Conventions of binary posts (subject lines, file identity), independent of AdonisJS.
|
||||
*/
|
||||
export { parseSubject, type ParsedSubject } from './parse.js'
|
||||
export { fileKey, weakFileKey, counterFileKey, looksLikeFilename } from './file_key.js'
|
||||
@@ -1,59 +0,0 @@
|
||||
export interface ParsedSubject {
|
||||
/** Filename from the quoted part of the subject. */
|
||||
name: string
|
||||
/** Segment number `k` from `(k/N)`. */
|
||||
segment: number
|
||||
/** Segment count `N` from `(k/N)`. */
|
||||
totalSegments: number
|
||||
/** File counter `[i/M]`, when present. */
|
||||
fileIndex?: number
|
||||
fileTotal?: number
|
||||
/**
|
||||
* The subject with the segment counter removed. Identical for every segment
|
||||
* of one uploaded file, so it is part of the file identity (see fileKey).
|
||||
*/
|
||||
base: string
|
||||
}
|
||||
|
||||
const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g
|
||||
const FILE_COUNTER = /\[(\d+)\/(\d+)\]/
|
||||
|
||||
/**
|
||||
* Parses a binary post subject such as
|
||||
* `[07/45] - "Some.Release.part06.rar" yEnc (12/137) 51315352`.
|
||||
*
|
||||
* Returns null for subjects without a quoted filename and segment counter
|
||||
* (obfuscated posts), which need their yEnc header read from the body instead.
|
||||
*/
|
||||
export function parseSubject(subject: string): ParsedSubject | null {
|
||||
const nameEnd = subject.lastIndexOf('"')
|
||||
const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
|
||||
if (nameStart === -1) return null
|
||||
const name = subject.slice(nameStart + 1, nameEnd)
|
||||
if (!name) return null
|
||||
|
||||
// The segment counter is the last "(k/N)" after the filename; a file size may follow it.
|
||||
let counter: RegExpExecArray | undefined
|
||||
for (const match of subject.matchAll(SEGMENT_COUNTER)) {
|
||||
if (match.index > nameEnd) counter = match
|
||||
}
|
||||
if (!counter) return null
|
||||
|
||||
const segment = Number.parseInt(counter[1], 10)
|
||||
const totalSegments = Number.parseInt(counter[2], 10)
|
||||
if (totalSegments < 1 || segment < 1 || segment > totalSegments) return null
|
||||
|
||||
const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
|
||||
const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart))
|
||||
return {
|
||||
name,
|
||||
segment,
|
||||
totalSegments,
|
||||
fileIndex: fileCounter ? Number.parseInt(fileCounter[1], 10) : undefined,
|
||||
fileTotal: fileCounter ? Number.parseInt(fileCounter[2], 10) : undefined,
|
||||
base,
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
import yencode from 'yencode'
|
||||
import { YencCrcError, YencError } from './errors.js'
|
||||
import { parseYencMeta, type YencMeta } from './meta.js'
|
||||
|
||||
export interface DecodedSegment {
|
||||
meta: YencMeta
|
||||
/** Decoded bytes of this segment. */
|
||||
data: Buffer
|
||||
/** 0-based offset of `data` within the complete file. */
|
||||
offset: number
|
||||
}
|
||||
|
||||
export interface DecodeOptions {
|
||||
/**
|
||||
* Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
|
||||
* bodies, so this defaults to true.
|
||||
*/
|
||||
stripDots?: boolean
|
||||
/** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
|
||||
verifyCrc?: boolean
|
||||
/**
|
||||
* Accept a decoded length that differs from the declared size (the CRC is
|
||||
* still checked). Some posting tools declare a size that leaves out a
|
||||
* header, e.g. the 20-byte header of `nzbe` containers.
|
||||
*/
|
||||
lenientSize?: boolean
|
||||
}
|
||||
|
||||
export function crc32Hex(data: Buffer): string {
|
||||
return yencode.crc32(data).toString('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
|
||||
* without the data itself.
|
||||
*/
|
||||
export function combineCrc32(parts: { crc: string; length: number }[]): string {
|
||||
let combined: Buffer | undefined
|
||||
for (const part of parts) {
|
||||
const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
|
||||
combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
|
||||
}
|
||||
return (combined ?? Buffer.alloc(4)).toString('hex')
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes one yEnc article body and checks its size and CRC32 against the
|
||||
* `=yend` trailer.
|
||||
*/
|
||||
export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
|
||||
const { stripDots = true, verifyCrc = true, lenientSize = false } = options
|
||||
const meta = parseYencMeta(body)
|
||||
const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
|
||||
|
||||
if (lenientSize) {
|
||||
// Only the CRC below decides.
|
||||
} else if (data.length !== meta.end.size) {
|
||||
throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
|
||||
}
|
||||
if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
|
||||
throw new YencError(
|
||||
`Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
|
||||
)
|
||||
}
|
||||
|
||||
if (verifyCrc) {
|
||||
// Multi-part posts carry pcrc32 per part; single-part posts only crc32.
|
||||
const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
|
||||
if (expected) {
|
||||
const actual = crc32Hex(data)
|
||||
if (actual !== expected) throw new YencCrcError(expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
export class YencError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'YencError'
|
||||
}
|
||||
}
|
||||
|
||||
export class YencCrcError extends YencError {
|
||||
constructor(
|
||||
public readonly expected: string,
|
||||
public readonly actual: string
|
||||
) {
|
||||
super(`CRC32 mismatch: expected ${expected}, got ${actual}`)
|
||||
this.name = 'YencCrcError'
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
/**
|
||||
* yEnc decoding, independent of AdonisJS. See CONCEPT.md (D4, D10).
|
||||
*/
|
||||
export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js'
|
||||
export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js'
|
||||
export { YencFile } from './yenc_file.js'
|
||||
export { YencError, YencCrcError } from './errors.js'
|
||||
@@ -1,140 +0,0 @@
|
||||
import { YencError } from './errors.js'
|
||||
|
||||
/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */
|
||||
export interface YencBegin {
|
||||
line: number
|
||||
size: number
|
||||
name: string
|
||||
part?: number
|
||||
total?: number
|
||||
}
|
||||
|
||||
/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */
|
||||
export interface YencPart {
|
||||
begin: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */
|
||||
export interface YencEnd {
|
||||
size: number
|
||||
part?: number
|
||||
pcrc32?: string
|
||||
crc32?: string
|
||||
}
|
||||
|
||||
export interface YencMeta {
|
||||
begin: YencBegin
|
||||
part?: YencPart
|
||||
end: YencEnd
|
||||
/** Byte range of the encoded data inside the article body. */
|
||||
dataStart: number
|
||||
dataEnd: number
|
||||
}
|
||||
|
||||
const CRLF = Buffer.from('\r\n')
|
||||
const BEGIN = Buffer.from('=ybegin ')
|
||||
const PART = Buffer.from('=ypart ')
|
||||
const END = Buffer.from('\r\n=yend ')
|
||||
|
||||
/**
|
||||
* Parses `key=value` pairs from a yEnc header line. `name` is always the last
|
||||
* field and runs to the end of the line, because filenames may contain spaces
|
||||
* and `=`.
|
||||
*/
|
||||
function parseFields(line: string): Record<string, string> {
|
||||
const fields: Record<string, string> = {}
|
||||
const nameIndex = line.search(/(^|\s)name=/)
|
||||
let rest = line
|
||||
if (nameIndex !== -1) {
|
||||
const nameStart = line.indexOf('name=', nameIndex) + 'name='.length
|
||||
fields.name = line.slice(nameStart).trim()
|
||||
rest = line.slice(0, nameIndex)
|
||||
}
|
||||
for (const match of rest.matchAll(/(\w+)=(\S+)/g)) {
|
||||
fields[match[1]] = match[2]
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
function toInt(fields: Record<string, string>, key: string, required: true): number
|
||||
function toInt(fields: Record<string, string>, key: string, required?: false): number | undefined
|
||||
function toInt(fields: Record<string, string>, key: string, required = false) {
|
||||
const raw = fields[key]
|
||||
if (raw === undefined) {
|
||||
if (required) throw new YencError(`Missing "${key}" in yEnc header`)
|
||||
return undefined
|
||||
}
|
||||
const value = Number.parseInt(raw, 10)
|
||||
if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`)
|
||||
return value
|
||||
}
|
||||
|
||||
function toCrc(fields: Record<string, string>, key: string) {
|
||||
const raw = fields[key]
|
||||
if (raw === undefined) return undefined
|
||||
if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`)
|
||||
return raw.toLowerCase().padStart(8, '0')
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of parts in the file. Many posters leave `total=` out of `=ybegin`;
|
||||
* then it follows from the part size, since all parts except the last one
|
||||
* are the same size.
|
||||
*/
|
||||
export function inferTotalParts(meta: Pick<YencMeta, 'begin' | 'part'>): number {
|
||||
const { begin, part } = meta
|
||||
if (begin.total) return begin.total
|
||||
if (!part || begin.part === undefined) return 1
|
||||
if (part.end >= begin.size) return begin.part
|
||||
return Math.ceil(begin.size / (part.end - part.begin + 1))
|
||||
}
|
||||
|
||||
function lineAt(buffer: Buffer, start: number) {
|
||||
const eol = buffer.indexOf(CRLF, start)
|
||||
const end = eol === -1 ? buffer.length : eol
|
||||
return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates and parses the yEnc header/trailer lines of one article body,
|
||||
* without decoding the data itself.
|
||||
*/
|
||||
export function parseYencMeta(body: Buffer): YencMeta {
|
||||
const beginIndex = body.indexOf(BEGIN)
|
||||
if (beginIndex === -1) throw new YencError('No =ybegin line found')
|
||||
|
||||
const beginLine = lineAt(body, beginIndex)
|
||||
const beginFields = parseFields(beginLine.text)
|
||||
const begin: YencBegin = {
|
||||
line: toInt(beginFields, 'line', true),
|
||||
size: toInt(beginFields, 'size', true),
|
||||
name: beginFields.name ?? '',
|
||||
part: toInt(beginFields, 'part'),
|
||||
total: toInt(beginFields, 'total'),
|
||||
}
|
||||
if (!begin.name) throw new YencError('Missing "name" in =ybegin line')
|
||||
|
||||
let dataStart = beginLine.next
|
||||
let part: YencPart | undefined
|
||||
if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) {
|
||||
const partLine = lineAt(body, dataStart)
|
||||
const partFields = parseFields(partLine.text)
|
||||
part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) }
|
||||
dataStart = partLine.next
|
||||
}
|
||||
|
||||
const endIndex = body.lastIndexOf(END)
|
||||
if (endIndex === -1 || endIndex < dataStart - CRLF.length) {
|
||||
throw new YencError('No =yend line found')
|
||||
}
|
||||
const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text)
|
||||
const end: YencEnd = {
|
||||
size: toInt(endFields, 'size', true),
|
||||
part: toInt(endFields, 'part'),
|
||||
pcrc32: toCrc(endFields, 'pcrc32'),
|
||||
crc32: toCrc(endFields, 'crc32'),
|
||||
}
|
||||
|
||||
return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) }
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
import { crc32Hex, type DecodedSegment } from './decode.js'
|
||||
import { YencCrcError, YencError } from './errors.js'
|
||||
import { inferTotalParts } from './meta.js'
|
||||
|
||||
/**
|
||||
* Assembles the decoded segments of one file into a single buffer. Segments
|
||||
* may be added in any order.
|
||||
*/
|
||||
export class YencFile {
|
||||
readonly name: string
|
||||
readonly size: number
|
||||
readonly totalParts: number
|
||||
|
||||
private readonly buffer: Buffer
|
||||
private readonly received = new Set<number>()
|
||||
private bytesWritten = 0
|
||||
private fileCrc?: string
|
||||
|
||||
constructor(name: string, size: number, totalParts = 1) {
|
||||
this.name = name
|
||||
this.size = size
|
||||
this.totalParts = totalParts
|
||||
this.buffer = Buffer.alloc(size)
|
||||
}
|
||||
|
||||
/** Creates a file sized from the metadata of any of its segments. */
|
||||
static fromSegment(segment: DecodedSegment): YencFile {
|
||||
const { begin } = segment.meta
|
||||
// Many posters leave total= out; it follows from the part size.
|
||||
return new YencFile(begin.name, begin.size, inferTotalParts(segment.meta))
|
||||
}
|
||||
|
||||
add(segment: DecodedSegment): void {
|
||||
const { meta, data, offset } = segment
|
||||
// Only the size is compared: obfuscating posters can give every segment
|
||||
// of one file a different random name.
|
||||
if (meta.begin.size !== this.size) {
|
||||
throw new YencError(
|
||||
`Segment of "${meta.begin.name}" (${meta.begin.size} bytes) does not belong to "${this.name}" (${this.size} bytes)`
|
||||
)
|
||||
}
|
||||
if (offset + data.length > this.size) {
|
||||
throw new YencError(`Segment at offset ${offset} overruns file size ${this.size}`)
|
||||
}
|
||||
|
||||
const partNumber = meta.begin.part ?? 1
|
||||
if (this.received.has(partNumber)) return
|
||||
|
||||
data.copy(this.buffer, offset)
|
||||
this.received.add(partNumber)
|
||||
this.bytesWritten += data.length
|
||||
if (meta.end.crc32) this.fileCrc = meta.end.crc32
|
||||
}
|
||||
|
||||
get partsReceived(): number {
|
||||
return this.received.size
|
||||
}
|
||||
|
||||
get isComplete(): boolean {
|
||||
return this.received.size === this.totalParts && this.bytesWritten === this.size
|
||||
}
|
||||
|
||||
/** Part numbers (1-based) that have not been added yet. */
|
||||
missingParts(): number[] {
|
||||
const missing: number[] = []
|
||||
for (let part = 1; part <= this.totalParts; part++) {
|
||||
if (!this.received.has(part)) missing.push(part)
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the assembled file. Throws if parts are missing or, when the poster
|
||||
* included a whole-file crc32, if it does not match.
|
||||
*/
|
||||
getBuffer(): Buffer {
|
||||
if (!this.isComplete) {
|
||||
throw new YencError(`File "${this.name}" is incomplete, missing parts: ${this.missingParts().join(', ')}`)
|
||||
}
|
||||
if (this.fileCrc) {
|
||||
const actual = crc32Hex(this.buffer)
|
||||
if (actual !== this.fileCrc) throw new YencCrcError(this.fileCrc, actual)
|
||||
}
|
||||
return this.buffer
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user