usenet-indexer/lib/spotnet/xml.ts

import { inflateRawSync } from 'node:zlib'

export interface SpotXml {
  title: string
  category: number | null
  subcats: string[]
  size: number
  website: string
  /** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */
  description: string
  tag: string
  /** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */
  image: SpotImage | null
  /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
  nzbSegments: string[]
  created: number | null
}

export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string }

const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }

/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */
function text(value: string | undefined): string {
  if (value === undefined) return ''
  const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
  return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
    if (entity[0] === '#') {
      const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
    }
    return ENTITIES[entity.toLowerCase()] ?? match
  })
}

const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, 'i').exec(xml)?.[1]

/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */
export function parseSpotXml(xml: string): SpotXml | null {
  const posting = element(xml, 'Posting')
  if (!posting) return null
  const categoryXml = element(posting, 'Category') ?? ''
  const key = Number.parseInt(element(posting, 'Key') ?? '', 10)
  const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
  const nzb = element(posting, 'NZB') ?? ''
  return {
    description: text(element(posting, 'Description')).trim(),
    tag: text(element(posting, 'Tag')).trim(),
    image: parseImage(posting),
    title: text(element(posting, 'Title')).trim(),
    // Newer keys number categories from 1.
    category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
    subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()),
    size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0,
    website: text(element(posting, 'Website')).trim(),
    nzbSegments: [...nzb.matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean),
    created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null,
  }
}

function parseImage(posting: string): SpotImage | null {
  const match = /<Image(\s[^>]*)?>([\s\S]*?)<\/Image>/i.exec(posting)
  if (!match) return null
  const segments = [...match[2].matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean)
  if (segments.length > 0) {
    const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined
    return { segments, width: size('Width'), height: size('Height') }
  }
  const url = text(match[2]).trim()
  return /^https?:\/\//i.test(url) ? { url } : null
}

/**
 * The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1;
 * the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8.
 */
export function decodeSpotText(latin1: string): string {
  const utf8 = Buffer.from(latin1, 'latin1').toString('utf8')
  return utf8.includes('\uFFFD') ? latin1 : utf8
}

/**
 * Header lines of a raw HEAD response; X-XML (split over many lines) joined.
 * Folded headers are unfolded: old (key 1) spots put the From address on a
 * continuation line.
 */
export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
  const headers: Record<string, string> = {}
  let xml = ''
  let last: string | null = null
  let lastIsFirst = false
  for (const line of raw.split(/\r?\n/)) {
    if (/^[ \t]/.test(line) && last) {
      if (last === 'x-xml') xml += line
      else if (lastIsFirst) headers[last] += line
      continue
    }
    const colon = line.indexOf(':')
    if (colon <= 0) {
      last = null
      continue
    }
    const name = line.slice(0, colon).toLowerCase()
    const value = line.slice(colon + 1).replace(/^ /, '')
    last = name
    lastIsFirst = !(name in headers)
    if (name === 'x-xml') xml += value
    else headers[name] ??= value
  }
  return { headers, xml }
}

/**
 * Spot binaries (the NZB, images): article bodies joined without line
 * breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw
 * deflate (PHP gzinflate).
 */
export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer {
  const joined = bodies
    .map((body) =>
      body
        .toString('latin1')
        .split('\r\n')
        // NNTP dot-stuffing.
        .map((line) => (line.startsWith('..') ? line.slice(1) : line))
        .join('')
    )
    .join('')
  const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1')
  return compressed ? inflateRawSync(unescaped) : unescaped
}