implemented a lot of features

lib/yenc/decode.ts

@@ -0,0 +1,76 @@
+import yencode from 'yencode'
+import { YencCrcError, YencError } from './errors.js'
+import { parseYencMeta, type YencMeta } from './meta.js'
+
+export interface DecodedSegment {
+  meta: YencMeta
+  /** Decoded bytes of this segment. */
+  data: Buffer
+  /** 0-based offset of `data` within the complete file. */
+  offset: number
+}
+
+export interface DecodeOptions {
+  /**
+   * Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw
+   * bodies, so this defaults to true.
+   */
+  stripDots?: boolean
+  /** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */
+  verifyCrc?: boolean
+  /**
+   * Accept a decoded length that differs from the declared size (the CRC is
+   * still checked). Some posting tools declare a size that leaves out a
+   * header, e.g. the 20-byte header of `nzbe` containers.
+   */
+  lenientSize?: boolean
+}
+
+export function crc32Hex(data: Buffer): string {
+  return yencode.crc32(data).toString('hex')
+}
+
+/**
+ * Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order,
+ * without the data itself.
+ */
+export function combineCrc32(parts: { crc: string; length: number }[]): string {
+  let combined: Buffer | undefined
+  for (const part of parts) {
+    const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex')
+    combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc
+  }
+  return (combined ?? Buffer.alloc(4)).toString('hex')
+}
+
+/**
+ * Decodes one yEnc article body and checks its size and CRC32 against the
+ * `=yend` trailer.
+ */
+export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment {
+  const { stripDots = true, verifyCrc = true, lenientSize = false } = options
+  const meta = parseYencMeta(body)
+  const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots)
+
+  if (lenientSize) {
+    // Only the CRC below decides.
+  } else if (data.length !== meta.end.size) {
+    throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`)
+  }
+  if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) {
+    throw new YencError(
+      `Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}`
+    )
+  }
+
+  if (verifyCrc) {
+    // Multi-part posts carry pcrc32 per part; single-part posts only crc32.
+    const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32)
+    if (expected) {
+      const actual = crc32Hex(data)
+      if (actual !== expected) throw new YencCrcError(expected, actual)
+    }
+  }
+
+  return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 }
+}

lib/yenc/errors.ts

@@ -0,0 +1,16 @@
+export class YencError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'YencError'
+  }
+}
+
+export class YencCrcError extends YencError {
+  constructor(
+    public readonly expected: string,
+    public readonly actual: string
+  ) {
+    super(`CRC32 mismatch: expected ${expected}, got ${actual}`)
+    this.name = 'YencCrcError'
+  }
+}

lib/yenc/index.ts

@@ -0,0 +1,7 @@
+/**
+ * yEnc decoding, independent of AdonisJS. See CONCEPT.md (D4, D10).
+ */
+export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js'
+export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js'
+export { YencFile } from './yenc_file.js'
+export { YencError, YencCrcError } from './errors.js'

lib/yenc/meta.ts

@@ -0,0 +1,140 @@
+import { YencError } from './errors.js'
+
+/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */
+export interface YencBegin {
+  line: number
+  size: number
+  name: string
+  part?: number
+  total?: number
+}
+
+/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */
+export interface YencPart {
+  begin: number
+  end: number
+}
+
+/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */
+export interface YencEnd {
+  size: number
+  part?: number
+  pcrc32?: string
+  crc32?: string
+}
+
+export interface YencMeta {
+  begin: YencBegin
+  part?: YencPart
+  end: YencEnd
+  /** Byte range of the encoded data inside the article body. */
+  dataStart: number
+  dataEnd: number
+}
+
+const CRLF = Buffer.from('\r\n')
+const BEGIN = Buffer.from('=ybegin ')
+const PART = Buffer.from('=ypart ')
+const END = Buffer.from('\r\n=yend ')
+
+/**
+ * Parses `key=value` pairs from a yEnc header line. `name` is always the last
+ * field and runs to the end of the line, because filenames may contain spaces
+ * and `=`.
+ */
+function parseFields(line: string): Record<string, string> {
+  const fields: Record<string, string> = {}
+  const nameIndex = line.search(/(^|\s)name=/)
+  let rest = line
+  if (nameIndex !== -1) {
+    const nameStart = line.indexOf('name=', nameIndex) + 'name='.length
+    fields.name = line.slice(nameStart).trim()
+    rest = line.slice(0, nameIndex)
+  }
+  for (const match of rest.matchAll(/(\w+)=(\S+)/g)) {
+    fields[match[1]] = match[2]
+  }
+  return fields
+}
+
+function toInt(fields: Record<string, string>, key: string, required: true): number
+function toInt(fields: Record<string, string>, key: string, required?: false): number | undefined
+function toInt(fields: Record<string, string>, key: string, required = false) {
+  const raw = fields[key]
+  if (raw === undefined) {
+    if (required) throw new YencError(`Missing "${key}" in yEnc header`)
+    return undefined
+  }
+  const value = Number.parseInt(raw, 10)
+  if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`)
+  return value
+}
+
+function toCrc(fields: Record<string, string>, key: string) {
+  const raw = fields[key]
+  if (raw === undefined) return undefined
+  if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`)
+  return raw.toLowerCase().padStart(8, '0')
+}
+
+/**
+ * Number of parts in the file. Many posters leave `total=` out of `=ybegin`;
+ * then it follows from the part size, since all parts except the last one
+ * are the same size.
+ */
+export function inferTotalParts(meta: Pick<YencMeta, 'begin' | 'part'>): number {
+  const { begin, part } = meta
+  if (begin.total) return begin.total
+  if (!part || begin.part === undefined) return 1
+  if (part.end >= begin.size) return begin.part
+  return Math.ceil(begin.size / (part.end - part.begin + 1))
+}
+
+function lineAt(buffer: Buffer, start: number) {
+  const eol = buffer.indexOf(CRLF, start)
+  const end = eol === -1 ? buffer.length : eol
+  return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length }
+}
+
+/**
+ * Locates and parses the yEnc header/trailer lines of one article body,
+ * without decoding the data itself.
+ */
+export function parseYencMeta(body: Buffer): YencMeta {
+  const beginIndex = body.indexOf(BEGIN)
+  if (beginIndex === -1) throw new YencError('No =ybegin line found')
+
+  const beginLine = lineAt(body, beginIndex)
+  const beginFields = parseFields(beginLine.text)
+  const begin: YencBegin = {
+    line: toInt(beginFields, 'line', true),
+    size: toInt(beginFields, 'size', true),
+    name: beginFields.name ?? '',
+    part: toInt(beginFields, 'part'),
+    total: toInt(beginFields, 'total'),
+  }
+  if (!begin.name) throw new YencError('Missing "name" in =ybegin line')
+
+  let dataStart = beginLine.next
+  let part: YencPart | undefined
+  if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) {
+    const partLine = lineAt(body, dataStart)
+    const partFields = parseFields(partLine.text)
+    part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) }
+    dataStart = partLine.next
+  }
+
+  const endIndex = body.lastIndexOf(END)
+  if (endIndex === -1 || endIndex < dataStart - CRLF.length) {
+    throw new YencError('No =yend line found')
+  }
+  const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text)
+  const end: YencEnd = {
+    size: toInt(endFields, 'size', true),
+    part: toInt(endFields, 'part'),
+    pcrc32: toCrc(endFields, 'pcrc32'),
+    crc32: toCrc(endFields, 'crc32'),
+  }
+
+  return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) }
+}

lib/yenc/yenc_file.ts

@@ -0,0 +1,86 @@
+import { crc32Hex, type DecodedSegment } from './decode.js'
+import { YencCrcError, YencError } from './errors.js'
+import { inferTotalParts } from './meta.js'
+
+/**
+ * Assembles the decoded segments of one file into a single buffer. Segments
+ * may be added in any order.
+ */
+export class YencFile {
+  readonly name: string
+  readonly size: number
+  readonly totalParts: number
+
+  private readonly buffer: Buffer
+  private readonly received = new Set<number>()
+  private bytesWritten = 0
+  private fileCrc?: string
+
+  constructor(name: string, size: number, totalParts = 1) {
+    this.name = name
+    this.size = size
+    this.totalParts = totalParts
+    this.buffer = Buffer.alloc(size)
+  }
+
+  /** Creates a file sized from the metadata of any of its segments. */
+  static fromSegment(segment: DecodedSegment): YencFile {
+    const { begin } = segment.meta
+    // Many posters leave total= out; it follows from the part size.
+    return new YencFile(begin.name, begin.size, inferTotalParts(segment.meta))
+  }
+
+  add(segment: DecodedSegment): void {
+    const { meta, data, offset } = segment
+    // Only the size is compared: obfuscating posters can give every segment
+    // of one file a different random name.
+    if (meta.begin.size !== this.size) {
+      throw new YencError(
+        `Segment of "${meta.begin.name}" (${meta.begin.size} bytes) does not belong to "${this.name}" (${this.size} bytes)`
+      )
+    }
+    if (offset + data.length > this.size) {
+      throw new YencError(`Segment at offset ${offset} overruns file size ${this.size}`)
+    }
+
+    const partNumber = meta.begin.part ?? 1
+    if (this.received.has(partNumber)) return
+
+    data.copy(this.buffer, offset)
+    this.received.add(partNumber)
+    this.bytesWritten += data.length
+    if (meta.end.crc32) this.fileCrc = meta.end.crc32
+  }
+
+  get partsReceived(): number {
+    return this.received.size
+  }
+
+  get isComplete(): boolean {
+    return this.received.size === this.totalParts && this.bytesWritten === this.size
+  }
+
+  /** Part numbers (1-based) that have not been added yet. */
+  missingParts(): number[] {
+    const missing: number[] = []
+    for (let part = 1; part <= this.totalParts; part++) {
+      if (!this.received.has(part)) missing.push(part)
+    }
+    return missing
+  }
+
+  /**
+   * Returns the assembled file. Throws if parts are missing or, when the poster
+   * included a whole-file crc32, if it does not match.
+   */
+  getBuffer(): Buffer {
+    if (!this.isComplete) {
+      throw new YencError(`File "${this.name}" is incomplete, missing parts: ${this.missingParts().join(', ')}`)
+    }
+    if (this.fileCrc) {
+      const actual = crc32Hex(this.buffer)
+      if (actual !== this.fileCrc) throw new YencCrcError(this.fileCrc, actual)
+    }
+    return this.buffer
+  }
+}