implemented a lot of features

lib/spotnet/header.ts

@@ -0,0 +1,162 @@
+import { createHash, createPublicKey, verify } from 'node:crypto'
+import { crc32 } from 'node:zlib'
+
+/**
+ * Spotnet spot headers (free.pt), after Spotweb's Services_Format_Parsing::parseHeader.
+ *
+ *   From: Nick <[PUBKEY[.USERSIGNATURE]]@[CAT][KEYID][SUBCATS].[SIZE].[x].[DATE].[..].[SIGNATURE]>
+ *   Subject: Title | Tag
+ */
+export interface SpotHeader {
+  messageId: string
+  poster: string
+  title: string
+  tag: string
+  /** 0 image/video, 1 sound, 2 games, 3 applications. */
+  category: number
+  keyId: number
+  /** e.g. { a: [9], b: [3], c: [7, 10], d: [17], z: [0] } */
+  subcats: Record<string, number[]>
+  size: number
+  /** Unix seconds. */
+  stamp: number
+  /** Everything after the @. */
+  header: string
+  headerSign: string
+  selfSignedPubKey: string
+  userSignature?: string
+}
+
+/** Spotnet's base64 variant: `-s` for `/`, `-p` for `+`, padding left out. */
+export function spotUnprepareBase64(input: string): string {
+  const padded = input.length % 4 ? input + '='.repeat(4 - (input.length % 4)) : input
+  return padded.replace(/-s/g, '/').replace(/-p/g, '+')
+}
+
+/** Parses the From/Subject of a spot. Null when it isn't one. */
+export function parseSpotHeader(from: string, subject: string, messageId: string, date?: string): SpotHeader | null {
+  const lt = from.indexOf('<')
+  if (lt === -1) return null
+  const address = from.slice(lt + 1, from.endsWith('>') ? -1 : undefined)
+  const at = address.lastIndexOf('@')
+  if (at === -1) return null
+  const header = address.slice(at + 1)
+  const [pubKey, userSignature] = address.slice(0, at).split('.')
+
+  const fields = header.split('.')
+  if (fields.length < 6) return null
+  const category = Number.parseInt(fields[0][0], 10) - 1
+  const keyId = Number.parseInt(fields[0][1], 10)
+  if (!(category >= 0) || !(keyId >= 0)) return null
+
+  // a9b4c0d5 (FTD) or a09b04c00d05 (newer): a letter, then digits.
+  const subcats: Record<string, number[]> = {}
+  for (const [, letter, digits] of fields[0].slice(2).toLowerCase().matchAll(/([a-z])(\d+)/g)) {
+    if ('abcdz'.includes(letter)) (subcats[letter] ??= []).push(Number.parseInt(digits, 10))
+  }
+
+  // Newer keys: "Title | Tag". Key 1 (oldest): the tag is the last field.
+  const parts = subject.split('|')
+  let title: string
+  let tag = ''
+  if (keyId !== 1) {
+    title = parts[0].trim()
+    tag = parts.length > 1 ? parts[1].trim() : ''
+  } else {
+    tag = parts.length > 1 ? parts[parts.length - 1].trim() : subject.trim()
+    title = parts.length > 1 ? parts.slice(0, -2).join('|').trim() : ''
+  }
+
+  const parsedDate = date ? Date.parse(date) : Number.NaN
+  return {
+    messageId: messageId.replace(/^<|>$/g, ''),
+    poster: from.slice(0, lt).trim(),
+    title,
+    tag,
+    category,
+    keyId,
+    subcats,
+    size: Number.parseInt(fields[1], 10) || 0,
+    stamp: Number.isNaN(parsedDate) ? Number.parseInt(fields[3], 10) || 0 : Math.floor(parsedDate / 1000),
+    header,
+    headerSign: fields[fields.length - 1],
+    selfSignedPubKey: spotUnprepareBase64(pubKey ?? ''),
+    userSignature: userSignature ? spotUnprepareBase64(userSignature) : undefined,
+  }
+}
+
+/** Spotnet's own signing keys (Spotweb settings `rsa_keys`), base64 modulus. */
+export const SPOTNET_KEYS: Record<number, string> = {
+  2: 'ys8WSlqonQMWT8ubG0tAA2Q07P36E+CJmb875wSR1XH7IFhEi0CCwlUzNqBFhC+P',
+  3: 'uiyChPV23eguLAJNttC/o0nAsxXgdjtvUvidV2JL+hjNzc4Tc/PPo2JdYvsqUsat',
+  4: '1k6RNDVD6yBYWR6kHmwzmSud7JkNV4SMigBrs+jFgOK5Ldzwl17mKXJhl+su/GR9',
+}
+
+const base64url = (b64: string) => Buffer.from(b64, 'base64').toString('base64url')
+
+/** RSA PKCS#1 v1.5 with SHA-1, as Spotweb's checkRsaSignature (exponent AQAB). */
+export function checkRsaSignature(message: Buffer | string, signatureB64: string, modulusB64: string): boolean {
+  try {
+    const key = createPublicKey({ key: { kty: 'RSA', n: base64url(modulusB64), e: 'AQAB' }, format: 'jwk' })
+    const data = typeof message === 'string' ? toBytes(message) : message
+    return verify('sha1', data, key, Buffer.from(signatureB64, 'base64'))
+  } catch {
+    return false
+  }
+}
+
+/** Spots are signed over their ISO-8859-1 bytes. */
+function toBytes(text: string): Buffer {
+  return /^[\u0000-ÿ]*$/.test(text) ? Buffer.from(text, 'latin1') : Buffer.from(text, 'utf8')
+}
+
+/** SPOTSIGN_V2 and self-signed messages carry a hashcash: sha1('<msgid>') starts with 0000. */
+function hashcash(messageId: string): boolean {
+  return createHash('sha1').update(`<${messageId}>`).digest('hex').startsWith('0000')
+}
+
+/**
+ * Verifies a spot header like Spotweb does: key 7 (self-signed) needs a
+ * hashcash and the poster's key, keys 2–4 Spotnet's keys; spots that also
+ * carry a user signature must have it match the message id.
+ */
+export function verifySpotHeader(spot: SpotHeader, keys: Record<number, string> = SPOTNET_KEYS): boolean {
+  if (!spot.title || !spot.poster) return false
+  // Old key 1 spots from before 2011 weren't signed.
+  if (spot.keyId === 1 && spot.stamp <= 1293870080) return true
+  if (!spot.headerSign) return false
+
+  const signature = spotUnprepareBase64(spot.headerSign)
+  const toCheck = spot.title + spot.header.slice(0, spot.header.length - spot.headerSign.length - 1) + spot.poster
+  let verified: boolean
+  if (spot.keyId === 7 || (spot.keyId === 2 && spot.selfSignedPubKey.length > 50)) {
+    // Self-signed (and key-2 "dispose" messages): the hashcash is the gate, as in Spotweb.
+    verified = hashcash(spot.messageId)
+  } else {
+    verified = keys[spot.keyId] !== undefined && checkRsaSignature(toCheck, signature, keys[spot.keyId])
+  }
+  if (verified && spot.userSignature && spot.selfSignedPubKey) {
+    verified = checkRsaSignature(`<${spot.messageId}>`, spot.userSignature, spot.selfSignedPubKey)
+  }
+  return verified
+}
+
+/** Spotweb's spotter id: base64 of the CRC32 (little-endian) of the poster's public key. */
+export function spotterId(pubKeyB64: string): string {
+  if (!pubKeyB64) return ''
+  const crc = Buffer.alloc(4)
+  crc.writeUInt32LE(crc32(Buffer.from(pubKeyB64, 'base64')) >>> 0)
+  return crc.toString('base64').replace(/[/+=]/g, '')
+}
+
+/**
+ * Moderation: key-2 spots titled "delete|dispose|remove <message-id>".
+ * From Spotnet's key they are moderator actions; with the poster's own key
+ * (a spotter id) a personal dispose of their own spot.
+ */
+export function moderationTarget(spot: Pick<SpotHeader, 'keyId' | 'title'>): string | null {
+  if (spot.keyId !== 2) return null
+  const [command, target] = spot.title.split(' ')
+  if (!['delete', 'dispose', 'remove'].includes(command?.toLowerCase()) || !target) return null
+  return target.replace(/^<|>$/g, '')
+}

lib/spotnet/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Spotnet (free.pt) spots, independent of AdonisJS: header parsing and
+ * signature checks after Spotweb, the spot XML and its compressed NZB.
+ * See CONCEPT.md (D29).
+ */
+export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js'
+export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, type SpotXml } from './xml.js'

lib/spotnet/xml.ts

@@ -0,0 +1,84 @@
+import { inflateRawSync } from 'node:zlib'
+
+export interface SpotXml {
+  title: string
+  category: number | null
+  subcats: string[]
+  size: number
+  website: string
+  /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */
+  nzbSegments: string[]
+  created: number | null
+}
+
+const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }
+
+function text(value: string | undefined): string {
+  if (value === undefined) return ''
+  const cdata = /^\s*<!\[CDATA\[([\s\S]*)\]\]>\s*$/.exec(value)
+  if (cdata) return cdata[1]
+  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
+    if (entity[0] === '#') {
+      const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10)
+      return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
+    }
+    return ENTITIES[entity.toLowerCase()] ?? match
+  })
+}
+
+const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, 'i').exec(xml)?.[1]
+
+/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */
+export function parseSpotXml(xml: string): SpotXml | null {
+  const posting = element(xml, 'Posting')
+  if (!posting) return null
+  const categoryXml = element(posting, 'Category') ?? ''
+  const key = Number.parseInt(element(posting, 'Key') ?? '', 10)
+  const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10)
+  const nzb = element(posting, 'NZB') ?? ''
+  return {
+    title: text(element(posting, 'Title')).trim(),
+    // Newer keys number categories from 1.
+    category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory,
+    subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()),
+    size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0,
+    website: text(element(posting, 'Website')).trim(),
+    nzbSegments: [...nzb.matchAll(/<Segment>([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean),
+    created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null,
+  }
+}
+
+/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */
+export function parseSpotArticleHeaders(raw: string): { headers: Record<string, string>; xml: string } {
+  const headers: Record<string, string> = {}
+  let xml = ''
+  for (const line of raw.split(/\r?\n/)) {
+    const colon = line.indexOf(':')
+    if (colon <= 0) continue
+    const name = line.slice(0, colon).toLowerCase()
+    const value = line.slice(colon + 1).replace(/^ /, '')
+    if (name === 'x-xml') xml += value
+    else headers[name] ??= value
+  }
+  return { headers, xml }
+}
+
+/**
+ * Spot binaries (the NZB, images): article bodies joined without line
+ * breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw
+ * deflate (PHP gzinflate).
+ */
+export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer {
+  const joined = bodies
+    .map((body) =>
+      body
+        .toString('latin1')
+        .split('\r\n')
+        // NNTP dot-stuffing.
+        .map((line) => (line.startsWith('..') ? line.slice(1) : line))
+        .join('')
+    )
+    .join('')
+  const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1')
+  return compressed ? inflateRawSync(unescaped) : unescaped
+}