diff --git a/app/services/prober.ts b/app/services/prober.ts new file mode 100644 index 0000000..fea8800 --- /dev/null +++ b/app/services/prober.ts @@ -0,0 +1,183 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import { setTimeout as sleep } from 'node:timers/promises' +import env from '#start/env' +import coordinatorConfig from '#config/coordinator' +import { MissingArticle, XoverClient } from '#lib/nntp/index' +import { YencError } from '#lib/yenc/index' +import { inspect, innerName, type Inspection } from '#lib/probe/index' + +interface Candidate { + id: number + releaseId: number + kind: string + messageId: string +} + +type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection } + +/** Kinds in the order they're probed: a random name's real name first, then par2 sets, then archives. */ +const KINDS = ['lone', 'par2', 'archive'] +const FLUSH_MS = 2000 +const REPORT_MS = 60_000 + +/** + * Phase 2's first step (REBUILD.md, part 6): reads the probe queue's + * articles (settled releases first, by kind) over PROBE_CONNECTIONS + * connections of its own, without compression, and stores what they show: + * per release and kind an Inspection, and on the release whether it's + * passworded and its inner name. One process: fetching (~0.1 s an article) + * dwarfs parsing (a few ms), so there's no queue between them. + */ +export class Prober { + private queue: Candidate[] = [] + private outcomes: Outcome[] = [] + private refilling: Promise | null = null + private stopped = false + private stats = { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, since: Date.now() } + + async run() { + // A previous run's claims: one prober at a time. + const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`) + if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`) + const connections = coordinatorConfig.probe.connections + logger.info(`Probing with ${connections} connections.`) + const flusher = (async () => { + while (!this.stopped || this.outcomes.length > 0) { + await sleep(FLUSH_MS) + await this.flush().catch((error) => logger.error(`Probing: storing results failed: ${error.message}`)) + this.report() + } + })() + await Promise.all(Array.from({ length: connections }, () => this.lane())) + // What was claimed and not probed goes back. + for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' }) + this.stopped = true + await flusher + } + + stop() { + this.stopped = true + } + + /** One connection's loop: an article at a time, reconnecting after a failure. */ + private async lane() { + let client: XoverClient | null = null + while (!this.stopped) { + const candidate = await this.next() + if (!candidate) { + await sleep(30_000) + continue + } + try { + client ??= await XoverClient.connect({ + host: env.get('NNTP_HOST')!, + port: env.get('NNTP_PORT', 563), + user: env.get('NNTP_USER'), + password: env.get('NNTP_PASSWORD'), + secure: env.get('NNTP_SECURE', true), + compress: false, + timeoutMs: coordinatorConfig.nntpTimeoutMs, + }) + const body = await client.body(candidate.messageId) + this.stats.articles++ + this.stats.bytes += body.length + try { + this.outcomes.push({ candidate, status: 'done', inspection: inspect(body) }) + } catch (error: any) { + // Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem. + if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`) + this.outcomes.push({ candidate, status: 'error' }) + } + } catch (error: any) { + if (error instanceof MissingArticle) { + // Taken down or expired: final (Eweka takes ~450 ms per missing article). + this.stats.articles++ + this.outcomes.push({ candidate, status: 'missing' }) + continue + } + logger.warn(`Probing ${candidate.messageId}: ${error.message}`) + this.outcomes.push({ candidate, status: 'waiting' }) + client?.close() + client = null + await sleep(5000) + } + } + client?.close() + } + + private async next(): Promise { + if (this.queue.length < coordinatorConfig.probe.batch / 2) { + this.refilling ??= this.claim().finally(() => (this.refilling = null)) + if (this.queue.length === 0) await this.refilling + } + return this.queue.shift() + } + + /** Waiting candidates of settled releases, by kind, claimed so another round doesn't take them. */ + private async claim() { + for (const kind of KINDS) { + const want = coordinatorConfig.probe.batch - this.queue.length + if (want <= 0) return + const { rows } = await db.rawQuery( + `UPDATE probe_candidates SET status = 'taken' WHERE id IN ( + SELECT pc.id FROM probe_candidates pc JOIN releases r ON r.id = pc.release_id + WHERE pc.status = 'waiting' AND pc.kind = ? AND r.settled_at IS NOT NULL + ORDER BY pc.id LIMIT ? FOR UPDATE OF pc SKIP LOCKED) + RETURNING id, release_id, kind, message_id`, + [kind, want] + ) + for (const r of rows) this.queue.push({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id }) + } + } + + /** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */ + private async flush() { + const outcomes = this.outcomes.splice(0) + if (outcomes.length === 0) return + for (const o of outcomes) if (o.status !== 'waiting') this.stats[o.status]++ + await db.transaction(async (trx) => { + await trx.rawQuery( + `UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`, + [JSON.stringify(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))] + ) + const done = outcomes.filter((o) => o.inspection) + if (done.length === 0) return + await trx.rawQuery( + `INSERT INTO release_inspections (release_id, kind, result) + SELECT v.release_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind text, result jsonb) + ON CONFLICT (release_id, kind) DO UPDATE SET result = excluded.result, created_at = now()`, + [JSON.stringify(done.map((o) => ({ release_id: o.candidate.releaseId, kind: o.candidate.kind, result: o.inspection })))] + ) + // Each touched release from all its inspections (earlier rounds may have stored its other kinds). + const ids = [...new Set(done.map((o) => o.candidate.releaseId))] + const { rows } = await trx.rawQuery(`SELECT release_id, kind, result FROM release_inspections WHERE release_id = ANY(?)`, [ids]) + const byRelease = new Map>() + for (const r of rows) { + const id = Number(r.release_id) + byRelease.set(id, { ...byRelease.get(id), [r.kind]: r.result }) + } + const updates = [...byRelease].map(([id, all]) => ({ + id, + passworded: Object.values(all).some((i) => i.passworded === true), + inner_name: innerName(all), + })) + await trx.rawQuery( + `UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now() + FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`, + [JSON.stringify(updates)] + ) + }) + } + + private report() { + const seconds = (Date.now() - this.stats.since) / 1000 + if (seconds < REPORT_MS / 1000) return + const { articles, bytes, done, missing, error } = this.stats + logger.info( + `Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` + + `${done} inspected, ${missing} missing, ${error} not yEnc.` + ) + this.stats = { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, since: Date.now() } + } +} diff --git a/commands/probe_run.ts b/commands/probe_run.ts new file mode 100644 index 0000000..e180359 --- /dev/null +++ b/commands/probe_run.ts @@ -0,0 +1,25 @@ +import { BaseCommand } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import { Prober } from '#services/prober' + +/** Phase 2's prober (REBUILD.md, part 6): reads the probe queue's articles and stores what they show. */ +export default class ProbeRun extends BaseCommand { + static commandName = 'probe:run' + static description = "Reads the probe queue's articles (PROBE_CONNECTIONS) and stores what they show" + static options: CommandOptions = { startApp: true, staysAlive: true } + + async run() { + const prober = new Prober() + let done: () => void + const finished = new Promise((resolve) => (done = resolve)) + this.app.terminating(async () => { + this.logger.info('Stopping: finishing the articles in flight.') + prober.stop() + await finished + }) + await prober.run() + this.logger.info('Prober stopped.') + done!() + await this.terminate() + } +} diff --git a/config/coordinator.ts b/config/coordinator.ts index 03ee2e0..6a16475 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -65,6 +65,16 @@ export default { * under 2 minutes old as on ones over 10 (~25–35%; LOG.md 2026-10-09 (16)). */ nntpConnectionMaxMs: env.get('NNTP_CONNECTION_MAX_MINUTES', 0) * 60 * 1000, + probe: { + /** + * NNTP connections the prober reads articles with, besides the + * coordinator's: the account has 50, the coordinator 40 + 1 plain, and a + * couple stay free for downloads. ~7 MB/s each (~10 articles/s). + */ + connections: env.get('PROBE_CONNECTIONS', 6), + /** Candidates claimed at a time. */ + batch: 200, + }, spool: { dir: env.get('SPOOL_DIR', app.tmpPath('spool')), /** Ranges kept fetched ahead, per active lease. */ diff --git a/database/migrations/1770000001700_release_inspections.ts b/database/migrations/1770000001700_release_inspections.ts new file mode 100644 index 0000000..ca7e776 --- /dev/null +++ b/database/migrations/1770000001700_release_inspections.ts @@ -0,0 +1,38 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' +import env from '#start/env' + +/** + * Phase 2's probe results (REBUILD.md, part 6): per release and probe kind + * what its article showed (yEnc name, content type, a few names inside, + * encryption), in CHUNK_TABLESPACE (the HDD) when set: they're read per + * release, not searched. On the catalog only what search needs: whether the + * release is passworded and its best inner name. + */ +export default class extends BaseSchema { + async up() { + const tablespace = env.get('CHUNK_TABLESPACE') + const where = tablespace ? ` TABLESPACE ${tablespace}` : '' + this.schema.raw(` + CREATE TABLE release_inspections ( + release_id bigint NOT NULL, + kind varchar(8) NOT NULL, + result jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (release_id, kind)${tablespace ? ` USING INDEX TABLESPACE ${tablespace}` : ''} + )${where}`) + this.schema.alterTable('releases', (table) => { + table.boolean('passworded').nullable() + table.string('inner_name', 255).nullable() + table.timestamp('inspected_at', { useTz: true }).nullable() + }) + } + + async down() { + this.schema.alterTable('releases', (table) => { + table.dropColumn('passworded') + table.dropColumn('inner_name') + table.dropColumn('inspected_at') + }) + this.schema.dropTable('release_inspections') + } +} diff --git a/deploy/README.md b/deploy/README.md index 6b8cbf6..7ccac07 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,6 +17,7 @@ homelab with the deploy key, whose forced command | `migrate` | `usenet-rebuild` (target `app`) | `migration:run`, before every deploy | | `web` | `usenet-rebuild` | node API, catalog API, Newznab, web interface; port 3334 | | `coordinator` | `usenet-rebuild` | prefetching (all NNTP connections), planning, the tail, merging, settling | +| `prober` | `usenet-rebuild` | phase 2: reads the probe queue's articles over `PROBE_CONNECTIONS` (6) connections, stores what they show | | `edge` | `usenet-rebuild-edge` (target `edge`) | the Rust worker, `--threads $EDGE_THREADS` (8) | | `edge-ts` | `usenet-rebuild` | the TypeScript worker, profile `edge-ts`, not started | | `ace` | `usenet-rebuild` | one-off ace commands, profile `tools` | @@ -62,6 +63,7 @@ docker compose up -d --no-deps edge coordinator - Stop `edge` before `web`: a worker holding a stream keeps the server from exiting. - The NNTP account is shared with voyager 1 and downloads: `NNTP_CONNECTIONS` - (40) plus one plain connection, out of 50. + (40) plus one plain connection, plus the prober's `PROBE_CONNECTIONS` (6), + out of 50. - `deploy.sh` doesn't update itself: after changing it, install it again (`install -m 755 src/deploy/deploy.sh ~/bin/deploy-usenet-rebuild`). diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 3e31613..69d7a78 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -82,6 +82,16 @@ services: <<: *app-env NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-40} + # Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS + # connections of its own (next to the coordinator's 40 + 1; the account has 50). + prober: + <<: *app + command: ['node', 'ace', 'probe:run'] + volumes: [] + environment: + <<: *app-env + PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-6} + # The Rust worker; it has no NNTP account and talks to web only. Stopped # before web (a worker holding a stream keeps the server from exiting). edge: diff --git a/deploy/deploy.sh b/deploy/deploy.sh index e1c3345..e5b7145 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -45,10 +45,10 @@ docker compose run --rm --no-deps migrate # stream keeps the web server from exiting. echo "== recreating app services" docker compose stop edge -docker compose up -d --no-deps web coordinator +docker compose up -d --no-deps web coordinator prober docker compose up -d --no-deps edge echo "$SHA $(date -Is)" >> "$ROOT/deployed.log" # The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor). docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null -docker compose ps --format '{{.Service}} {{.Status}}' web coordinator edge +docker compose ps --format '{{.Service}} {{.Status}}' web coordinator prober edge echo "== deployed $SHA" diff --git a/lib/content/index.ts b/lib/content/index.ts new file mode 100644 index 0000000..d373af3 --- /dev/null +++ b/lib/content/index.ts @@ -0,0 +1,2 @@ +/** Content recognition by magic bytes (from voyager 1's lib/content). */ +export { sniffContentType, type ContentType } from './sniff.js' diff --git a/lib/content/sniff.ts b/lib/content/sniff.ts new file mode 100644 index 0000000..ebd15b9 --- /dev/null +++ b/lib/content/sniff.ts @@ -0,0 +1,81 @@ +/** + * Recognises a file type from its first bytes, independent of AdonisJS. + * `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)): + * "nzbe", 16 random bytes (salt/IV), then ciphertext. + * `yencbin` is the encrypted `.bin` container of `yEncBin@Poster.com` + * (LOG.md 2026-09-30 (3)): a fixed header, then data with 8 bits/byte of entropy. + */ +export type ContentType = + /** Not sniffed: the server no longer has the file's first article (inspection records it, no retries). */ + | 'missing' + | 'nzbe' + | 'par2' + | 'rar4' + | 'rar5' + | '7z' + | 'zip' + | 'zstd' + | 'gzip' + | 'xz' + | 'matroska' + | 'mp4' + | 'rclone' + | 'yencbin' + | 'iso' + | 'udf' + | 'pdf' + | 'nzb' + | 'jpeg' + | 'png' + | 'mp3' + | 'flac' + | 'avi' + | 'wav' + | 'ogg' + | 'bzip2' + +/** Magic bytes at an offset; null matches any byte. */ +const SIGNATURES: [ContentType, number, (number | null)[]][] = [ + // Two header variants seen (54f399d117c49269 6e4dc4ef5b60c11d…, 54f399e917c49269 6a4e9fb05b60c31d…): + // the bytes they share. + ['yencbin', 0, [0x54, 0xf3, 0x99, null, 0x17, 0xc4, 0x92, 0x69, null, null, null, null, 0x5b, 0x60, null, 0x1d, 0x7f, 0x83, 0xe9, 0x2d]], + ['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]], + ['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]], + ['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]], + ['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]], + ['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]], + ['zip', 0, [0x50, 0x4b, 0x03, 0x04]], + ['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]], + ['gzip', 0, [0x1f, 0x8b]], + ['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]], + ['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]], + ['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box + ['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt + ['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]], + ['jpeg', 0, [0xff, 0xd8, 0xff]], + ['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]], + ['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag + ['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC" + ['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS" + ['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh" +] + +export function sniffContentType(data: Buffer): ContentType | null { + for (const [type, offset, bytes] of SIGNATURES) { + if (data.length >= offset + bytes.length && bytes.every((b, i) => b === null || data[offset + i] === b)) return type + } + // Disc images, visible when the first part is big enough (sector 16 at 32 KiB): + // ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03. + if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso' + if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' && + /^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf' + // RIFF container: AVI or WAV by form type. + if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') { + const form = data.subarray(8, 12).toString('latin1') + if (form === 'AVI ') return 'avi' + if (form === 'WAVE') return 'wav' + } + const head = data.subarray(0, 512).toString('latin1') + if (/^\s*(<\?xml[^>]*>\s*)?(])/.test(head)) return 'nzb' + return null +} diff --git a/lib/nntp/index.ts b/lib/nntp/index.ts index 6dea896..5f93745 100644 --- a/lib/nntp/index.ts +++ b/lib/nntp/index.ts @@ -1,4 +1,4 @@ -export { XoverClient, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js' +export { XoverClient, MissingArticle, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js' export { parseXrefGroups } from './xref.js' export { parseGroupResponse, type GroupInfo } from './group.js' export { hedge, type Request, type Hedged } from './hedge.js' diff --git a/lib/nntp/xover_client.ts b/lib/nntp/xover_client.ts index 846e6e0..7e2bbf1 100644 --- a/lib/nntp/xover_client.ts +++ b/lib/nntp/xover_client.ts @@ -34,6 +34,9 @@ const DOT_LINE = Buffer.from('.\r\n') * followed by a plain "." line; an empty range is a 420 without body. * About six times less traffic per header (LOG.md 2026-10-05). */ +/** The server doesn't have the article (430): taken down or expired. */ +export class MissingArticle extends Error {} + export class XoverClient { private chunks: Buffer[] = [] private length = 0 @@ -228,6 +231,37 @@ export class XoverClient { } } + /** + * An article's body by message-id, raw (dot-stuffed, "\r\n.\r\n" cut off): + * yEnc decoding undoes the stuffing. A connection without COMPRESS GZIP. + * 430 (no such article) throws MissingArticle. + */ + async body(messageId: string): Promise { + return this.timed(async () => { + const id = messageId.startsWith('<') ? messageId : `<${messageId}>` + const reply = await this.command(`BODY ${id}`) + if (reply.startsWith('430') || reply.startsWith('423')) throw new MissingArticle(reply) + await this.expect(reply, ['222']) + return this.rawBody() + }) + } + + /** A multi-line body up to "\r\n.\r\n", as bytes, the terminator cut off. */ + private async rawBody(): Promise { + let searched = 0 + while (true) { + const all = this.buffer() + if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) { + this.take(3) + return Buffer.alloc(0) + } + const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length)) + if (at !== -1) return Buffer.from(this.take(at + TERMINATOR.length).subarray(0, at + 2)) + searched = all.length + await this.more() + } + } + /** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */ private async plainBody(): Promise { let searched = 0 diff --git a/lib/par2/index.ts b/lib/par2/index.ts new file mode 100644 index 0000000..bc2d916 --- /dev/null +++ b/lib/par2/index.ts @@ -0,0 +1,2 @@ +/** Par2 FileDesc packets (from voyager 1's lib/par2): the files a par2 set covers. */ +export { parsePar2, isPar2, type Par2File, type Par2Set } from './parse.js' diff --git a/lib/par2/parse.ts b/lib/par2/parse.ts new file mode 100644 index 0000000..6a30e79 --- /dev/null +++ b/lib/par2/parse.ts @@ -0,0 +1,74 @@ +import { createHash } from 'node:crypto' + +export interface Par2File { + /** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */ + fileId: string + /** MD5 of the whole file (hex). */ + md5: string + /** MD5 of the first 16 KiB (hex). */ + md5_16k: string + length: number + name: string +} + +export interface Par2Set { + /** Recovery set ID (hex), shared by every packet of one par2 set. */ + recoverySetId: string + files: Par2File[] +} + +const MAGIC = Buffer.from('PAR2\0PKT', 'latin1') +const HEADER_SIZE = 64 +const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1') + +export function isPar2(data: Buffer): boolean { + return data.subarray(0, MAGIC.length).equals(MAGIC) +} + +/** + * Reads the FileDesc packets of a par2 file (index or recovery volume). + * Packets with a bad length or checksum are skipped; parsing resumes at the + * next magic sequence. Returns null when no valid FileDesc packet is found. + */ +export function parsePar2(data: Buffer): Par2Set | null { + const files = new Map() + let recoverySetId: string | undefined + let offset = data.indexOf(MAGIC) + + while (offset !== -1 && offset + HEADER_SIZE <= data.length) { + const length = Number(data.readBigUInt64LE(offset + 8)) + const valid = + length >= HEADER_SIZE && + length % 4 === 0 && + offset + length <= data.length && + createHash('md5') + .update(data.subarray(offset + 32, offset + length)) + .digest() + .equals(data.subarray(offset + 16, offset + 32)) + + if (!valid) { + offset = data.indexOf(MAGIC, offset + 1) + continue + } + + const setId = data.subarray(offset + 32, offset + 48).toString('hex') + const type = data.subarray(offset + 48, offset + 64) + if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) { + recoverySetId = setId + const body = data.subarray(offset + HEADER_SIZE, offset + length) + const fileId = body.subarray(0, 16).toString('hex') + files.set(fileId, { + fileId, + md5: body.subarray(16, 32).toString('hex'), + md5_16k: body.subarray(32, 48).toString('hex'), + length: Number(body.readBigUInt64LE(48)), + // The name is padded with NULs to a multiple of 4 bytes. + name: body.subarray(56).toString('utf8').replace(/\0+$/, ''), + }) + } + offset += length + } + + if (!recoverySetId || files.size === 0) return null + return { recoverySetId, files: [...files.values()] } +} diff --git a/lib/probe/index.ts b/lib/probe/index.ts new file mode 100644 index 0000000..79351c2 --- /dev/null +++ b/lib/probe/index.ts @@ -0,0 +1 @@ +export { inspect, innerName, MAX_NAMES, type Inspection } from './inspect.js' diff --git a/lib/probe/inspect.ts b/lib/probe/inspect.ts new file mode 100644 index 0000000..0f67946 --- /dev/null +++ b/lib/probe/inspect.ts @@ -0,0 +1,79 @@ +import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index' +import { isPar2, parsePar2 } from '#lib/par2/index' +import { parseRarHeaders } from '#lib/rar/index' +import { sniffContentType, type ContentType } from '#lib/content/index' + +/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */ +export interface Inspection { + /** The file name in the yEnc header. */ + yencName: string + /** The whole file's size, from the yEnc header. */ + size: number + type: ContentType | null + /** Names inside: an archive's entries or a par2 set's files, the largest first, at most MAX_NAMES. */ + names?: string[] + /** How many entries or files there are in all. */ + entries?: number + /** An archive with encrypted entries or headers. */ + passworded?: boolean +} + +/** Names kept per inspection: the catalog wants the content's name, not every volume's. */ +export const MAX_NAMES = 8 +const MAX_NAME_LENGTH = 200 + +/** Archive volumes, par2 volumes and split parts: names of the packaging, not the content. */ +const PACKAGING = /\.(rar|r\d{2,3}|s\d{2}|\d{3}|7z(\.\d+)?|zip|z\d{2}|par2|sfv|nfo|nzb)$/i + +/** Decodes a body; some tools declare a size that leaves out a header, the CRC still has to match. */ +function decodeBody(body: Buffer): DecodedSegment { + try { + return decodeSegment(body) + } catch (error) { + if (!(error instanceof YencError) || error instanceof YencCrcError) throw error + return decodeSegment(body, { lenientSize: true }) + } +} + +function largestFirst(items: { name: string; size: number }[]): string[] { + return items + .sort((a, b) => b.size - a.size) + .slice(0, MAX_NAMES) + .map((i) => i.name.slice(0, MAX_NAME_LENGTH)) +} + +/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */ +export function inspect(body: Buffer): Inspection { + const segment = decodeBody(body) + const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) } + if (isPar2(segment.data)) { + // An index file fits one article; what a larger one cut off is skipped by the parser. + const set = parsePar2(segment.data) + if (set) { + result.entries = set.files.length + result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length }))) + } + return result + } + if (result.type === 'rar4' || result.type === 'rar5') { + const info = parseRarHeaders(segment.data) + if (info) { + const files = info.entries.filter((e) => !e.directory) + result.entries = files.length + result.names = largestFirst(files.map((e) => ({ name: e.name, size: Number(e.size ?? 0) }))) + result.passworded = info.headersEncrypted || files.some((e) => e.encrypted) + } + } + return result +} + +/** + * A release's best inner name from its inspections: the largest named entry + * of its archive, else the largest file of its par2 set that isn't + * packaging, else a lone file's yEnc name. Null when none says more. + */ +export function innerName(inspections: Partial>): string | null { + const content = (i?: Inspection) => i?.names?.find((n) => !PACKAGING.test(n)) + const pick = content(inspections.archive) ?? content(inspections.par2) ?? (inspections.lone?.yencName || undefined) + return pick ? pick.replace(/^.*[\\/]/, '').slice(0, 255) : null +} diff --git a/lib/rar/index.ts b/lib/rar/index.ts new file mode 100644 index 0000000..c936768 --- /dev/null +++ b/lib/rar/index.ts @@ -0,0 +1,2 @@ +/** RAR 4 and 5 headers (from voyager 1's lib/rar): a volume's listing, encryption, volume flags. */ +export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js' diff --git a/lib/rar/parse.ts b/lib/rar/parse.ts new file mode 100644 index 0000000..915e6bc --- /dev/null +++ b/lib/rar/parse.ts @@ -0,0 +1,351 @@ +import { crc32 } from 'node:zlib' + +export interface RarEntry { + name: string + /** Unpacked size in bytes (null when the archiver did not record it). */ + size: number | null + packedSize: number + directory: boolean + /** The file's data is password protected. */ + encrypted: boolean + /** Continues from the previous volume. */ + splitBefore: boolean + /** Continues in the next volume. */ + splitAfter: boolean + /** Stored without compression, so the data can be copied out as is. */ + stored: boolean + /** Offset of the entry's (packed) data in the parsed buffer. */ + dataOffset: number +} + +export interface RarInfo { + /** 7z listings take the same shape (see lib/sevenzip). */ + format: 'rar4' | 'rar5' | '7z' + /** Part of a multi-volume set. */ + volume: boolean + /** First volume of the set (null when the format version does not say). */ + firstVolume: boolean | null + solid: boolean + /** File names are encrypted too; nothing can be listed without the password. */ + headersEncrypted: boolean + entries: RarEntry[] + /** The buffer ended before the end-of-archive header (normal for one segment). */ + truncated: boolean + /** + * Offset of the first block that was not read: where the next header + * starts (possibly beyond the buffer, after an entry's data), or the + * header the buffer cut off. After the end-of-archive header, its end. + */ + next: number +} + +const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]) +const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) + +export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null { + if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5' + if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4' + return null +} + +/** + * Lists a RAR archive from its headers only, so the first segment of a first + * volume is enough. Returns null when the data is not a RAR archive. + * + * With `continueAs`, `data` starts at a header in the middle of a volume (no + * signature), e.g. where a previous parse said the next header is. + */ +export function parseRarHeaders(data: Buffer, continueAs?: 'rar4' | 'rar5'): RarInfo | null { + if (continueAs === 'rar4') return parseRar4(data, 0) + if (continueAs === 'rar5') return parseRar5(data, 0) + const format = rarFormat(data) + if (format === 'rar4') return parseRar4(data, RAR4_MARKER.length) + if (format === 'rar5') return parseRar5(data, RAR5_MARKER.length) + return null +} + +// --- RAR 4.x (RAR 1.5 – 4.x archive format) ------------------------------- + +const R4_MAIN = 0x73 +const R4_FILE = 0x74 +const R4_END = 0x7b +const R4_LONG_BLOCK = 0x8000 + +function parseRar4(data: Buffer, start: number): RarInfo { + const info: RarInfo = { + format: 'rar4', + volume: false, + firstVolume: null, + solid: false, + headersEncrypted: false, + entries: [], + truncated: true, + next: start, + } + let p = start + + while (p + 7 <= data.length) { + const headCrc = data.readUInt16LE(p) + const type = data[p + 2] + const flags = data.readUInt16LE(p + 3) + const size = data.readUInt16LE(p + 5) + if (size < 7 || p + size > data.length) break + + if (type === R4_END) { + info.truncated = false + p += size + break + } + + let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0) + + if (type === R4_MAIN) { + info.volume = Boolean(flags & 0x0001) + info.solid = Boolean(flags & 0x0008) + info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true + if (flags & 0x0080) { + info.headersEncrypted = true + break + } + } else if (type === R4_FILE) { + if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break + const large = Boolean(flags & 0x0100) + let packed = data.readUInt32LE(p + 7) + let unpacked = data.readUInt32LE(p + 11) + const nameSize = data.readUInt16LE(p + 26) + let nameStart = p + 32 + if (large) { + packed += data.readUInt32LE(p + 32) * 2 ** 32 + unpacked += data.readUInt32LE(p + 36) * 2 ** 32 + nameStart = p + 40 + } + const rawName = data.subarray(nameStart, nameStart + nameSize) + info.entries.push({ + name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'), + // 0xffffffff with LHD_LARGE unset means "unknown" in some archivers. + size: unpacked === 0xffffffff && !large ? null : unpacked, + packedSize: packed, + directory: (flags & 0x00e0) === 0x00e0, + encrypted: Boolean(flags & 0x0004), + splitBefore: Boolean(flags & 0x0001), + splitAfter: Boolean(flags & 0x0002), + stored: data[p + 25] === 0x30, + dataOffset: p + size, + }) + next = p + size + packed + } + + if (next <= p) break + p = next + } + // Archivers before RAR 3.0 don't set the first-volume flag; a first + // volume is one whose first file does not continue from a previous one. + if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) { + info.firstVolume = true + } + info.next = p + return info +} + +/** + * RAR 4 stores Unicode names as "ASCII\0", where the + * compressed part refers back to the ASCII bytes (unrar's DecodeFileName). + */ +export function decodeRar4Name(raw: Buffer): string { + const zero = raw.indexOf(0) + if (zero === -1) return raw.toString('utf8') + const ascii = raw.subarray(0, zero) + const enc = raw.subarray(zero + 1) + if (enc.length === 0) return ascii.toString('latin1') + + const out: number[] = [] + const highByte = enc[0] + let encPos = 1 + let flags = 0 + let flagBits = 0 + while (encPos < enc.length) { + if (flagBits === 0) { + flags = enc[encPos++] + flagBits = 8 + } + switch (flags >> 6) { + case 0: + out.push(enc[encPos++]) + break + case 1: + out.push(enc[encPos++] + (highByte << 8)) + break + case 2: + out.push(enc[encPos] + (enc[encPos + 1] << 8)) + encPos += 2 + break + case 3: { + let length = enc[encPos++] + if (length & 0x80) { + const correction = enc[encPos++] + for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) { + out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8)) + } + } else { + for (length += 2; length > 0 && out.length < ascii.length; length--) { + out.push(ascii[out.length]) + } + } + break + } + } + flags = (flags << 2) & 0xff + flagBits -= 2 + } + return String.fromCharCode(...out) +} + +// --- RAR 5.0 ---------------------------------------------------------------- + +const R5_MAIN = 1 +const R5_FILE = 2 +const R5_ENCRYPTION = 4 +const R5_END = 5 + +/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */ +function vint(data: Buffer, offset: number): { value: number; next: number } | null { + let value = 0 + let shift = 1 + for (let i = offset; i < data.length && i < offset + 10; i++) { + value += (data[i] & 0x7f) * shift + shift *= 128 + if (!(data[i] & 0x80)) return { value, next: i + 1 } + } + return null +} + +function parseRar5(data: Buffer, start: number): RarInfo { + const info: RarInfo = { + format: 'rar5', + volume: false, + firstVolume: null, + solid: false, + headersEncrypted: false, + entries: [], + truncated: true, + next: start, + } + let p = start + + while (p + 5 <= data.length) { + const headCrc = data.readUInt32LE(p) + const headerSize = vint(data, p + 4) + if (!headerSize || headerSize.value === 0) break + const headerEnd = headerSize.next + headerSize.value + if (headerEnd > data.length) break + if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break + + const type = vint(data, headerSize.next) + const flags = type && vint(data, type.next) + if (!type || !flags) break + let cursor = flags.next + let extraSize = 0 + let dataSize = 0 + if (flags.value & 0x0001) { + const v = vint(data, cursor) + if (!v) break + extraSize = v.value + cursor = v.next + } + if (flags.value & 0x0002) { + const v = vint(data, cursor) + if (!v) break + dataSize = v.value + cursor = v.next + } + + if (type.value === R5_END) { + info.truncated = false + p = headerEnd + break + } + if (type.value === R5_ENCRYPTION) { + info.headersEncrypted = true + break + } + + if (type.value === R5_MAIN) { + const archiveFlags = vint(data, cursor) + if (!archiveFlags) break + info.volume = Boolean(archiveFlags.value & 0x0001) + // A volume number is only stored for volumes after the first. + info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true + info.solid = Boolean(archiveFlags.value & 0x0004) + } else if (type.value === R5_FILE) { + const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize) + if (!entry) break + info.entries.push(entry) + } + + const next = headerEnd + dataSize + if (next <= p) break + p = next + } + info.next = p + return info +} + +function parseRar5File( + data: Buffer, + start: number, + extraStart: number, + headerEnd: number, + headerFlags: number, + dataSize: number +): RarEntry | null { + const fileFlags = vint(data, start) + const unpacked = fileFlags && vint(data, fileFlags.next) + const attributes = unpacked && vint(data, unpacked.next) + if (!fileFlags || !unpacked || !attributes) return null + let cursor = attributes.next + if (fileFlags.value & 0x0002) cursor += 4 // mtime + if (fileFlags.value & 0x0004) cursor += 4 // data CRC32 + const compression = vint(data, cursor) + const hostOs = compression && vint(data, compression.next) + const nameLength = hostOs && vint(data, hostOs.next) + if (!compression || !hostOs || !nameLength) return null + const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8') + // Compression information: bits 7-9 are the method, 0 is "store". + const method = Math.floor(compression.value / 128) & 0x7 + + // Extra area records: size, type, data. Type 1 is file encryption. + let encrypted = false + let e = extraStart + while (e < headerEnd) { + const recordSize = vint(data, e) + const recordType = recordSize && vint(data, recordSize.next) + if (!recordSize || !recordType || recordSize.value === 0) break + if (recordType.value === 0x01) encrypted = true + e = recordSize.next + recordSize.value + } + + return { + name, + size: fileFlags.value & 0x0008 ? null : unpacked.value, + packedSize: dataSize, + directory: Boolean(fileFlags.value & 0x0001), + encrypted, + splitBefore: Boolean(headerFlags & 0x0008), + splitAfter: Boolean(headerFlags & 0x0010), + stored: method === 0, + dataOffset: headerEnd, + } +} + +/** + * The data of an entry stored without compression, copied out of the + * archive buffer. Null when it is compressed, encrypted, split over volumes + * or not completely in the buffer. + */ +export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null { + if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null + if (entry.size === null || entry.size !== entry.packedSize) return null + const end = entry.dataOffset + entry.packedSize + if (end > data.length) return null + return data.subarray(entry.dataOffset, end) +} diff --git a/lib/yenc/decode.ts b/lib/yenc/decode.ts new file mode 100644 index 0000000..60e5dc0 --- /dev/null +++ b/lib/yenc/decode.ts @@ -0,0 +1,76 @@ +import yencode from 'yencode' +import { YencCrcError, YencError } from './errors.js' +import { parseYencMeta, type YencMeta } from './meta.js' + +export interface DecodedSegment { + meta: YencMeta + /** Decoded bytes of this segment. */ + data: Buffer + /** 0-based offset of `data` within the complete file. */ + offset: number +} + +export interface DecodeOptions { + /** + * Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw + * bodies, so this defaults to true. + */ + stripDots?: boolean + /** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */ + verifyCrc?: boolean + /** + * Accept a decoded length that differs from the declared size (the CRC is + * still checked). Some posting tools declare a size that leaves out a + * header, e.g. the 20-byte header of `nzbe` containers. + */ + lenientSize?: boolean +} + +export function crc32Hex(data: Buffer): string { + return yencode.crc32(data).toString('hex') +} + +/** + * Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order, + * without the data itself. + */ +export function combineCrc32(parts: { crc: string; length: number }[]): string { + let combined: Buffer | undefined + for (const part of parts) { + const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex') + combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc + } + return (combined ?? Buffer.alloc(4)).toString('hex') +} + +/** + * Decodes one yEnc article body and checks its size and CRC32 against the + * `=yend` trailer. + */ +export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment { + const { stripDots = true, verifyCrc = true, lenientSize = false } = options + const meta = parseYencMeta(body) + const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots) + + if (lenientSize) { + // Only the CRC below decides. + } else if (data.length !== meta.end.size) { + throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`) + } + if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) { + throw new YencError( + `Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}` + ) + } + + if (verifyCrc) { + // Multi-part posts carry pcrc32 per part; single-part posts only crc32. + const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32) + if (expected) { + const actual = crc32Hex(data) + if (actual !== expected) throw new YencCrcError(expected, actual) + } + } + + return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 } +} diff --git a/lib/yenc/errors.ts b/lib/yenc/errors.ts new file mode 100644 index 0000000..bc95c63 --- /dev/null +++ b/lib/yenc/errors.ts @@ -0,0 +1,16 @@ +export class YencError extends Error { + constructor(message: string) { + super(message) + this.name = 'YencError' + } +} + +export class YencCrcError extends YencError { + constructor( + public readonly expected: string, + public readonly actual: string + ) { + super(`CRC32 mismatch: expected ${expected}, got ${actual}`) + this.name = 'YencCrcError' + } +} diff --git a/lib/yenc/index.ts b/lib/yenc/index.ts new file mode 100644 index 0000000..f7b5593 --- /dev/null +++ b/lib/yenc/index.ts @@ -0,0 +1,4 @@ +/** yEnc article bodies (from voyager 1's lib/yenc): header and trailer, decoding with size and CRC checks. */ +export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js' +export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js' +export { YencError, YencCrcError } from './errors.js' diff --git a/lib/yenc/meta.ts b/lib/yenc/meta.ts new file mode 100644 index 0000000..3046b6c --- /dev/null +++ b/lib/yenc/meta.ts @@ -0,0 +1,140 @@ +import { YencError } from './errors.js' + +/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */ +export interface YencBegin { + line: number + size: number + name: string + part?: number + total?: number +} + +/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */ +export interface YencPart { + begin: number + end: number +} + +/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */ +export interface YencEnd { + size: number + part?: number + pcrc32?: string + crc32?: string +} + +export interface YencMeta { + begin: YencBegin + part?: YencPart + end: YencEnd + /** Byte range of the encoded data inside the article body. */ + dataStart: number + dataEnd: number +} + +const CRLF = Buffer.from('\r\n') +const BEGIN = Buffer.from('=ybegin ') +const PART = Buffer.from('=ypart ') +const END = Buffer.from('\r\n=yend ') + +/** + * Parses `key=value` pairs from a yEnc header line. `name` is always the last + * field and runs to the end of the line, because filenames may contain spaces + * and `=`. + */ +function parseFields(line: string): Record { + const fields: Record = {} + const nameIndex = line.search(/(^|\s)name=/) + let rest = line + if (nameIndex !== -1) { + const nameStart = line.indexOf('name=', nameIndex) + 'name='.length + fields.name = line.slice(nameStart).trim() + rest = line.slice(0, nameIndex) + } + for (const match of rest.matchAll(/(\w+)=(\S+)/g)) { + fields[match[1]] = match[2] + } + return fields +} + +function toInt(fields: Record, key: string, required: true): number +function toInt(fields: Record, key: string, required?: false): number | undefined +function toInt(fields: Record, key: string, required = false) { + const raw = fields[key] + if (raw === undefined) { + if (required) throw new YencError(`Missing "${key}" in yEnc header`) + return undefined + } + const value = Number.parseInt(raw, 10) + if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`) + return value +} + +function toCrc(fields: Record, key: string) { + const raw = fields[key] + if (raw === undefined) return undefined + if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`) + return raw.toLowerCase().padStart(8, '0') +} + +/** + * Number of parts in the file. Many posters leave `total=` out of `=ybegin`; + * then it follows from the part size, since all parts except the last one + * are the same size. + */ +export function inferTotalParts(meta: Pick): number { + const { begin, part } = meta + if (begin.total) return begin.total + if (!part || begin.part === undefined) return 1 + if (part.end >= begin.size) return begin.part + return Math.ceil(begin.size / (part.end - part.begin + 1)) +} + +function lineAt(buffer: Buffer, start: number) { + const eol = buffer.indexOf(CRLF, start) + const end = eol === -1 ? buffer.length : eol + return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length } +} + +/** + * Locates and parses the yEnc header/trailer lines of one article body, + * without decoding the data itself. + */ +export function parseYencMeta(body: Buffer): YencMeta { + const beginIndex = body.indexOf(BEGIN) + if (beginIndex === -1) throw new YencError('No =ybegin line found') + + const beginLine = lineAt(body, beginIndex) + const beginFields = parseFields(beginLine.text) + const begin: YencBegin = { + line: toInt(beginFields, 'line', true), + size: toInt(beginFields, 'size', true), + name: beginFields.name ?? '', + part: toInt(beginFields, 'part'), + total: toInt(beginFields, 'total'), + } + if (!begin.name) throw new YencError('Missing "name" in =ybegin line') + + let dataStart = beginLine.next + let part: YencPart | undefined + if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) { + const partLine = lineAt(body, dataStart) + const partFields = parseFields(partLine.text) + part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) } + dataStart = partLine.next + } + + const endIndex = body.lastIndexOf(END) + if (endIndex === -1 || endIndex < dataStart - CRLF.length) { + throw new YencError('No =yend line found') + } + const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text) + const end: YencEnd = { + size: toInt(endFields, 'size', true), + part: toInt(endFields, 'part'), + pcrc32: toCrc(endFields, 'pcrc32'), + crc32: toCrc(endFields, 'crc32'), + } + + return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) } +} diff --git a/package-lock.json b/package-lock.json index 5dd5fbd..6da3161 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,8 @@ "luxon": "^3.7.2", "pg": "^8.23.1", "reflect-metadata": "^0.2.2", - "xmlbuilder2": "^4.0.3" + "xmlbuilder2": "^4.0.3", + "yencode": "^1.2.4" }, "devDependencies": { "@adonisjs/assembler": "^8.5.2", @@ -7364,6 +7365,16 @@ "node": "^20.19.0 || ^22.12.0 || >=23" } }, + "node_modules/yencode": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/yencode/-/yencode-1.2.4.tgz", + "integrity": "sha512-AfQkwlxmgfoWq04ATRUaA6YSjpclGzHNRt2BP76DogqvuiNUtuIj1Fq/fAThQmOwEXWlbQ6sMc/qjdeFtyCScQ==", + "hasInstallScript": true, + "license": "CC0-1.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 68da3d1..91c02ff 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,8 @@ "luxon": "^3.7.2", "pg": "^8.23.1", "reflect-metadata": "^0.2.2", - "xmlbuilder2": "^4.0.3" + "xmlbuilder2": "^4.0.3", + "yencode": "^1.2.4" }, "hotHook": { "boundaries": [ diff --git a/start/env.ts b/start/env.ts index cd83f7e..86bab59 100644 --- a/start/env.ts +++ b/start/env.ts @@ -64,5 +64,6 @@ export default await Env.create(new URL('../', import.meta.url), { SPOOL_MIN_AHEAD: Env.schema.number.optional(), SPOOL_MAX_GB: Env.schema.number.optional(), PREFETCH_PER_RANGE: Env.schema.number.optional(), + PROBE_CONNECTIONS: Env.schema.number.optional(), PREFETCH_PER_GROUP: Env.schema.number.optional(), }) diff --git a/tests/unit/nntp/xover_client.spec.ts b/tests/unit/nntp/xover_client.spec.ts index a8fa821..a651329 100644 --- a/tests/unit/nntp/xover_client.spec.ts +++ b/tests/unit/nntp/xover_client.spec.ts @@ -1,7 +1,7 @@ import { test } from '@japa/runner' import net from 'node:net' import { deflateSync } from 'node:zlib' -import { XoverClient, parseOverviews } from '#lib/nntp/index' +import { MissingArticle, XoverClient, parseOverviews } from '#lib/nntp/index' import { unzipSync } from 'node:zlib' const LINES = [ @@ -49,6 +49,11 @@ function server(): Promise<{ port: number; close: () => void }> { sock.write('215 NewsGroups Follow\r\n') send(Buffer.concat([body, Buffer.from('.\r\n')])) } + } else if (line === 'BODY ') sock.write('430 No Such Article\r\n') + else if (line.startsWith('BODY')) { + sock.write('222 0 body\r\n') + // Raw bytes, a dot-stuffed line among them. + send(Buffer.concat([Buffer.from([0x3d, 0x79, 0xff, 0x00, 0x0d, 0x0a]), Buffer.from('..stuffed\r\nlast\r\n.\r\n')])) } else if (line === 'QUIT') sock.end('205 bye\r\n') } }) @@ -58,6 +63,22 @@ function server(): Promise<{ port: number; close: () => void }> { } test.group('nntp / XoverClient', () => { + test('reads a body by message-id as bytes, raw; 430 is MissingArticle', async ({ assert }) => { + const { port, close } = await server() + const client = await XoverClient.connect({ host: '127.0.0.1', port, user: 'u', password: 'p', compress: false }) + try { + const body = await client.body('one@test') + assert.deepEqual([...body.subarray(0, 6)], [0x3d, 0x79, 0xff, 0x00, 0x0d, 0x0a]) + assert.equal(body.subarray(6).toString('latin1'), '..stuffed\r\nlast\r\n') + await assert.rejects(() => client.body(''), MissingArticle) + // The connection is still usable after both. + assert.equal((await client.body('')).length, body.length) + } finally { + client.close() + close() + } + }) + test('parses overview lines into the fields nntp-js gives', ({ assert }) => { const [[number, fields]] = parseOverviews(LINES[0] + '\r\n') assert.equal(number, 101) diff --git a/tests/unit/probe/inspect.spec.ts b/tests/unit/probe/inspect.spec.ts new file mode 100644 index 0000000..992beb0 --- /dev/null +++ b/tests/unit/probe/inspect.spec.ts @@ -0,0 +1,124 @@ +import { test } from '@japa/runner' +import { createHash } from 'node:crypto' +import { crc32 } from 'node:zlib' +import yencode from 'yencode' +import { crc32Hex } from '#lib/yenc/index' +import { inspect, innerName, MAX_NAMES } from '#lib/probe/index' + +/** A single-part yEnc article of `data`. */ +function article(data: Buffer, name: string) { + const lines = [`=ybegin line=128 size=${data.length} name=${name}`, yencode.encode(data).toString('latin1'), `=yend size=${data.length} crc32=${crc32Hex(data)}`] + return Buffer.from(lines.join('\r\n'), 'latin1') +} + +const md5 = (data: Buffer) => createHash('md5').update(data).digest() +/** Builds one PAR2 packet with a valid checksum. */ +function packet(setId: Buffer, type: string, body: Buffer) { + const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)]) + const length = Buffer.alloc(8) + length.writeBigUInt64LE(BigInt(64 + padded.length)) + const typeBuf = Buffer.alloc(16) + typeBuf.write(type, 'latin1') + const hashed = Buffer.concat([setId, typeBuf, padded]) + return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed]) +} +function fileDesc(setId: Buffer, name: string, size: number) { + const len = Buffer.alloc(8) + len.writeBigUInt64LE(BigInt(size)) + const md5_16k = md5(Buffer.from(`${name}-16k`)) + const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)])) + const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')]) + return packet(setId, 'PAR 2.0\0FileDesc', body) +} +const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) +function vint(value: number) { + const bytes: number[] = [] + do { + let byte = value % 128 + value = Math.floor(value / 128) + if (value > 0) byte |= 0x80 + bytes.push(byte) + } while (value > 0) + return Buffer.from(bytes) +} +function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) { + let flags = headerFlags + const parts = [vint(type)] + const sizes: Buffer[] = [] + if (opts.extra) { + flags |= 0x0001 + sizes.push(vint(opts.extra.length)) + } + if (opts.data !== undefined) { + flags |= 0x0002 + sizes.push(vint(opts.data)) + } + const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)]) + const sized = Buffer.concat([vint(header.length), header]) + const crc = Buffer.alloc(4) + crc.writeUInt32LE(crc32(sized) >>> 0) + return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)]) +} +function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) { + const nameBuf = Buffer.from(name, 'utf8') + const fields = Buffer.concat([ + vint(opts.dir ? 0x0001 : 0), // file flags + vint(size), + vint(0x20), // attributes + vint(0), // compression info + vint(0), // host OS + vint(nameBuf.length), + nameBuf, + ]) + // Encryption record: size, type 1, then its data (dummy here). + const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined + return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed }) +} + +test.group('probe / inspect', () => { + test('a par2 index: its files, largest first', ({ assert }) => { + const setId = Buffer.alloc(16, 3) + const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)]) + const result = inspect(article(par2, 'a8f3k2.par2')) + assert.deepEqual(result, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] }) + }) + + test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => { + const rar = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('Movie.2020.1080p/movie.mkv', 5_000_000, { packed: 10 }), rar5File('Movie.2020.1080p/sample.mkv', 1000, { packed: 10 })]) + const open = inspect(article(rar, 'x7q.part01.rar')) + assert.include(open, { type: 'rar5', entries: 2, passworded: false }) + assert.deepEqual(open.names, ['Movie.2020.1080p/movie.mkv', 'Movie.2020.1080p/sample.mkv']) + const locked = inspect(article(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]), 'y.rar')) + assert.isTrue(locked.passworded) + const hidden = inspect(article(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]), 'z.rar')) + assert.include(hidden, { passworded: true, entries: 0 }) + }) + + test('other data: the yEnc name and what the bytes are', ({ assert }) => { + const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)]) + assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska' }) + }) + + test('not yEnc: throws', ({ assert }) => { + assert.throws(() => inspect(Buffer.from('just some text\r\n'))) + }) + + test('at most MAX_NAMES names', ({ assert }) => { + const setId = Buffer.alloc(16, 4) + const par2 = Buffer.concat(Array.from({ length: MAX_NAMES + 5 }, (_, i) => fileDesc(setId, `f${i}.rar`, i))) + const result = inspect(article(par2, 'p.par2')) + assert.equal(result.entries, MAX_NAMES + 5) + assert.lengthOf(result.names!, MAX_NAMES) + }) +}) + +test.group('probe / innerName', () => { + test("the archive's largest entry, else par2's largest non-packaging file, else a lone file's yEnc name", ({ assert }) => { + const archive = { yencName: 'a.rar', size: 1, type: 'rar5' as const, names: ['Show.S01E01/show.s01e01.mkv'] } + const par2 = { yencName: 'a.par2', size: 1, type: 'par2' as const, names: ['abc.part01.rar', 'Show.S01E01.1080p.mkv'] } + assert.equal(innerName({ archive, par2 }), 'show.s01e01.mkv') + assert.equal(innerName({ par2 }), 'Show.S01E01.1080p.mkv') + assert.equal(innerName({ par2: { ...par2, names: ['abc.part01.rar', 'abc.par2'] } }), null) + assert.equal(innerName({ lone: { yencName: 'Real.Name.mkv', size: 1, type: 'matroska' } }), 'Real.Name.mkv') + }) +}) diff --git a/tests/unit/probe/par2.spec.ts b/tests/unit/probe/par2.spec.ts new file mode 100644 index 0000000..d6e2119 --- /dev/null +++ b/tests/unit/probe/par2.spec.ts @@ -0,0 +1,60 @@ +import { test } from '@japa/runner' +import { createHash } from 'node:crypto' +import { isPar2, parsePar2 } from '#lib/par2/index' + +const md5 = (data: Buffer) => createHash('md5').update(data).digest() + +/** Builds one PAR2 packet with a valid checksum. */ +function packet(setId: Buffer, type: string, body: Buffer) { + const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)]) + const length = Buffer.alloc(8) + length.writeBigUInt64LE(BigInt(64 + padded.length)) + const typeBuf = Buffer.alloc(16) + typeBuf.write(type, 'latin1') + const hashed = Buffer.concat([setId, typeBuf, padded]) + return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed]) +} + +function fileDesc(setId: Buffer, name: string, size: number) { + const len = Buffer.alloc(8) + len.writeBigUInt64LE(BigInt(size)) + const md5_16k = md5(Buffer.from(`${name}-16k`)) + const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)])) + const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')]) + return packet(setId, 'PAR 2.0\0FileDesc', body) +} + +test.group('par2 / parsePar2', () => { + const setId = Buffer.alloc(16, 7) + + test('reads file names and sizes from FileDesc packets', ({ assert }) => { + const data = Buffer.concat([ + packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)), + fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), + fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800), + fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common + ]) + assert.isTrue(isPar2(data)) + const set = parsePar2(data)! + assert.equal(set.recoverySetId, setId.toString('hex')) + assert.deepEqual( + set.files.map((f) => [f.name, f.length]), + [ + ['Some.Movie.2024.part01.rar', 104857600], + ['Some.Movie.2024.part02.rar', 52428800], + ] + ) + }) + + test('skips corrupted packets and resumes at the next one', ({ assert }) => { + const bad = fileDesc(setId, 'bad.bin', 1) + bad[100] ^= 0xff + const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)]) + assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin']) + }) + + test('returns null for non-par2 data', ({ assert }) => { + assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1'))) + assert.isNull(parsePar2(Buffer.from('hello world'))) + }) +}) diff --git a/tests/unit/probe/rar.spec.ts b/tests/unit/probe/rar.spec.ts new file mode 100644 index 0000000..62f237d --- /dev/null +++ b/tests/unit/probe/rar.spec.ts @@ -0,0 +1,189 @@ +import { test } from '@japa/runner' +import { crc32 } from 'node:zlib' +import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, type RarEntry } from '#lib/rar/index' + +/** Entries without the extraction fields, for comparing listings. */ +const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest) + +// --- RAR 4 builders ----------------------------------------------------------- + +const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]) + +function rar4Block(type: number, flags: number, body: Buffer) { + const header = Buffer.alloc(7) + header.writeUInt8(type, 2) + header.writeUInt16LE(flags, 3) + header.writeUInt16LE(7 + body.length, 5) + const block = Buffer.concat([header, body]) + block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0) + return block +} + +function rar4Main(flags: number) { + return rar4Block(0x73, flags, Buffer.alloc(6)) +} + +function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) { + const body = Buffer.alloc(25) + body.writeUInt8(opts.method ?? 0x33, 18) + body.writeUInt32LE(opts.packed, 0) + body.writeUInt32LE(opts.unpacked, 4) + body.writeUInt16LE(name.length, 19) + // LONG_BLOCK (0x8000) is always set on file headers. + return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)]) +} + +// --- RAR 5 builders ----------------------------------------------------------- + +const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) + +function vint(value: number) { + const bytes: number[] = [] + do { + let byte = value % 128 + value = Math.floor(value / 128) + if (value > 0) byte |= 0x80 + bytes.push(byte) + } while (value > 0) + return Buffer.from(bytes) +} + +function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) { + let flags = headerFlags + const parts = [vint(type)] + const sizes: Buffer[] = [] + if (opts.extra) { + flags |= 0x0001 + sizes.push(vint(opts.extra.length)) + } + if (opts.data !== undefined) { + flags |= 0x0002 + sizes.push(vint(opts.data)) + } + const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)]) + const sized = Buffer.concat([vint(header.length), header]) + const crc = Buffer.alloc(4) + crc.writeUInt32LE(crc32(sized) >>> 0) + return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)]) +} + +function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) { + const nameBuf = Buffer.from(name, 'utf8') + const fields = Buffer.concat([ + vint(opts.dir ? 0x0001 : 0), // file flags + vint(size), + vint(0x20), // attributes + vint(0), // compression info + vint(0), // host OS + vint(nameBuf.length), + nameBuf, + ]) + // Encryption record: size, type 1, then its data (dummy here). + const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined + return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed }) +} + +test.group('rar / RAR 4', () => { + test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => { + const data = Buffer.concat([ + RAR4, + rar4Main(0x0001 | 0x0100), // volume, first volume + rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }), + ]) + assert.equal(rarFormat(data), 'rar4') + const info = parseRarHeaders(data)! + assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true }) + assert.deepEqual(listing(info.entries), [ + { name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true }, + ]) + }) + + test('detects password-protected files and encrypted headers', ({ assert }) => { + const files = parseRarHeaders( + Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })]) + )! + assert.isTrue(files.entries[0].encrypted) + const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))! + assert.isTrue(headers.headersEncrypted) + assert.lengthOf(headers.entries, 0) + }) + + test('copies out stored entries, not compressed ones', ({ assert }) => { + const data = Buffer.concat([ + RAR4, + rar4Main(0), + rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }), + rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }), + ]) + const [stored, packed] = parseRarHeaders(data)!.entries + data.write('', stored.dataOffset) + assert.equal(storedEntryData(data, stored)?.toString(), '') + assert.isFalse(packed.stored) + assert.isNull(storedEntryData(data, packed)) + }) + + test('stops at a header with a bad CRC', ({ assert }) => { + const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 }) + const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 }) + bad[10] ^= 0xff + const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))! + assert.deepEqual(info.entries.map((e) => e.name), ['a.bin']) + }) + + test('decodes compressed Unicode names', ({ assert }) => { + // "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte, + // then 3 plain bytes), followed by one more flag byte for the last byte. + const ascii = Buffer.from('?.txt', 'latin1') + const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74]) + assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt') + assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt') + }) +}) + +test.group('rar / RAR 5', () => { + test('lists files and reads volume flags', ({ assert }) => { + const data = Buffer.concat([ + RAR5, + rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume + rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }), + ]) + assert.equal(rarFormat(data), 'rar5') + const info = parseRarHeaders(data)! + assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false }) + assert.deepEqual(listing(info.entries), [ + { name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true }, + ]) + }) + + test('a later volume carries a volume number', ({ assert }) => { + const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))! + assert.isFalse(info.firstVolume) + }) + + test('detects encrypted files and encrypted headers', ({ assert }) => { + const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))! + assert.isTrue(files.entries[0].encrypted) + const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))! + assert.isTrue(headers.headersEncrypted) + }) + + test('marks the listing complete at the end-of-archive header', ({ assert }) => { + const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))! + assert.isFalse(info.truncated) + assert.lengthOf(info.entries, 1) + }) + + test('copies out entries stored without compression', ({ assert }) => { + const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))]) + const entry = parseRarHeaders(data)!.entries[0] + assert.isTrue(entry.stored) + data.write('', entry.dataOffset) + assert.equal(storedEntryData(data, entry)?.toString(), '') + assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry)) + assert.isNull(storedEntryData(data, { ...entry, encrypted: true })) + }) + + test('returns null for other data', ({ assert }) => { + assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1'))) + }) +}) diff --git a/tests/unit/probe/yenc.spec.ts b/tests/unit/probe/yenc.spec.ts new file mode 100644 index 0000000..f09d6a5 --- /dev/null +++ b/tests/unit/probe/yenc.spec.ts @@ -0,0 +1,107 @@ +import { test } from '@japa/runner' +import yencode from 'yencode' +import { + YencCrcError, + YencError, + crc32Hex, + decodeSegment, + parseYencMeta, +} from '#lib/yenc/index' + +/** Builds an article body as a yEnc poster would send it. */ +function article( + data: Buffer, + opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string } +) { + const fileSize = opts.fileSize ?? data.length + const encoded = yencode.encode(data).toString('latin1') + const lines: string[] = [] + if (opts.part) { + lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`) + const begin = opts.begin ?? 1 + lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`) + lines.push(encoded) + const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : '' + lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`) + } else { + lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`) + lines.push(encoded) + lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`) + } + return Buffer.from(lines.join('\r\n'), 'latin1') +} + +function randomBytes(length: number, seed = 1) { + const buf = Buffer.alloc(length) + let x = seed + for (let i = 0; i < length; i++) { + x = (x * 1103515245 + 12345) & 0x7fffffff + buf[i] = x & 0xff + } + return buf +} + +test.group('yenc / parseYencMeta', () => { + test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => { + const body = Buffer.from( + '=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' + + '=ypart begin=21120001 end=21504000\r\n' + + 'xyz\r\n' + + '=yend size=384000 part=56 pcrc32=B6339AFC', + 'latin1' + ) + const meta = parseYencMeta(body) + assert.deepEqual(meta.begin, { + part: 56, + total: undefined, + line: 128, + size: 104857600, + name: 'obrMkrDeLmbhaouwYO34.part08.rar', + }) + assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 }) + assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined }) + assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz') + }) + + test('keeps spaces and "=" in the filename', ({ assert }) => { + const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' }) + assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo') + }) + + test('ignores leading blank lines before =ybegin', ({ assert }) => { + const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })]) + assert.equal(parseYencMeta(body).begin.name, 'a.txt') + }) + + test('rejects bodies without yEnc markers', ({ assert }) => { + assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/) + assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/) + }) +}) + +test.group('yenc / decodeSegment', () => { + test('round-trips a single-part post and checks crc32', ({ assert }) => { + const data = randomBytes(5000) + const segment = decodeSegment(article(data, { name: 'single.bin' })) + assert.isTrue(segment.data.equals(data)) + assert.equal(segment.offset, 0) + }) + + test('throws YencCrcError on a corrupted part', ({ assert }) => { + const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 }) + const index = body.indexOf('\r\n=ypart') + 60 + body[index] = body[index] === 0x41 ? 0x42 : 0x41 + assert.throws(() => decodeSegment(body), YencCrcError as any) + }) + + test('undoes NNTP dot-stuffing', ({ assert }) => { + // Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..". + const data = Buffer.from([4, 5, 6]) + const body = Buffer.from( + `=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`, + 'latin1' + ) + assert.isTrue(decodeSegment(body).data.equals(data)) + assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any) + }) +}) diff --git a/types/yencode.d.ts b/types/yencode.d.ts new file mode 100644 index 0000000..aff38db --- /dev/null +++ b/types/yencode.d.ts @@ -0,0 +1,15 @@ +declare module 'yencode' { + export function decode(data: Buffer, stripDots?: boolean): Buffer + export function encode(data: Buffer, lineSize?: number, column?: number): Buffer + /** Returns the CRC32 as a 4-byte big-endian buffer. */ + export function crc32(data: Buffer, initial?: Buffer): Buffer + /** CRC32 of A+B from crc(A), crc(B) and the length of B. */ + export function crc32_combine(crc1: Buffer, crc2: Buffer, len2: number): Buffer + const yencode: { + decode: typeof decode + encode: typeof encode + crc32: typeof crc32 + crc32_combine: typeof crc32_combine + } + export default yencode +}