usenet-indexer/lib/chunks/codec.ts

import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib'

/** A file of a release chunk: what an NZB needs, once per file, and its segments as [part, message-id, bytes]. */
export interface ChunkFile {
  name: string
  poster: string
  subject: string
  /** Earliest post date of its articles, ms since 1970 (null when no article had a valid date). */
  postedAt: number | null
  groups: string[]
  segments: [number, string, number][]
}

/**
 * Message-ids of the two tools that post most (LOG.md 2026-10-08 (3)):
 * nyuu's 12 random capital+small letter pairs, a ms timestamp and a fixed
 * suffix; ngPost-style 32 random hex digits and a fixed suffix. Their random
 * parts are stored as bits (zstd can't shrink random text much), the rest
 * as text.
 */
const NYUU_BYTES = 15
const HEX_BYTES = 16

const zstd = (data: Buffer) => zstdCompressSync(data, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })

/**
 * A release chunk's payload (REBUILD.md, chunk store), version 2: a version
 * byte, the length of the compressed part (4 bytes), then zstd of a JSON
 * line with the files (name, poster, subject, date, groups, sizes, part
 * numbers unless 1..N, and per file how its ids are kept) and the other
 * files' message-ids one per line, then the raw bytes of the nyuu and hex
 * ids. ~25 → ~18 B per segment on boneless (LOG.md 2026-10-09).
 */
export function encodeChunk(files: ChunkFile[]): Buffer {
  const text: string[] = []
  // First how each file's ids are kept, then all raw ids into one buffer (a buffer per id was slow).
  const codes: ('n' | 'h' | 't')[] = files.map((f) => {
    const ids = f.segments
    const first = ids[0]?.[1]
    if (!first) return 't'
    const nyuuSuffix = first.slice(38)
    if (nyuuSuffix.startsWith('@') && ids.every(([, id]) => isNyuu(id, nyuuSuffix))) return 'n'
    const hexSuffix = first.slice(32)
    if (hexSuffix.startsWith('@') && ids.every(([, id]) => isHex(id, hexSuffix))) return 'h'
    return 't'
  })
  let rawBytes = 0
  files.forEach((f, i) => (rawBytes += codes[i] === 'n' ? NYUU_BYTES * f.segments.length : codes[i] === 'h' ? HEX_BYTES * f.segments.length : 0))
  const raw = Buffer.allocUnsafe(rawBytes)
  let at = 0
  const head = files.map((f, i) => {
    const sequential = f.segments.every(([n], k) => n === k + 1)
    const meta: Record<string, unknown> = {
      n: f.name,
      p: f.poster,
      j: f.subject,
      d: f.postedAt,
      g: f.groups,
      s: f.segments.map(([, , b]) => b),
      ...(sequential ? {} : { k: f.segments.map(([n]) => n) }),
    }
    if (codes[i] === 'n') {
      let previous = 0
      const deltas: number[] = []
      for (const [, id] of f.segments) {
        packLetters(id, raw, at)
        at += NYUU_BYTES
        const t = Number(id.slice(25, 38))
        deltas.push(t - previous)
        previous = t
      }
      return { ...meta, c: 'n', x: f.segments[0][1].slice(38), t: deltas }
    }
    if (codes[i] === 'h') {
      for (const [, id] of f.segments) at += raw.write(id.slice(0, 32), at, 'hex')
      return { ...meta, c: 'h', x: f.segments[0][1].slice(32) }
    }
    for (const [, id] of f.segments) text.push(id)
    return meta
  })
  const packed = zstd(Buffer.from([JSON.stringify(head), ...text].join('\n'), 'utf8'))
  const prefix = Buffer.alloc(5)
  prefix[0] = 2
  prefix.writeUInt32BE(packed.length, 1)
  return Buffer.concat([prefix, packed, raw])
}

export function decodeChunk(data: Buffer): ChunkFile[] {
  if (data[0] === 1) return decodeV1(data)
  if (data[0] !== 2) throw new Error(`Unknown chunk version ${data[0]}`)
  const length = data.readUInt32BE(1)
  const [headLine, ...text] = zstdDecompressSync(data.subarray(5, 5 + length)).toString('utf8').split('\n')
  let at = 5 + length
  let line = 0
  return (JSON.parse(headLine) as any[]).map((f) => {
    const sizes = f.s as number[]
    let ids: string[]
    if (f.c === 'n') {
      let t = 0
      ids = sizes.map((_, i) => {
        t += f.t[i]
        const letters = unpackLetters(data.subarray(at, at + NYUU_BYTES))
        at += NYUU_BYTES
        return `${letters}-${String(t).padStart(13, '0')}${f.x}`
      })
    } else if (f.c === 'h') {
      ids = sizes.map(() => {
        const id = data.subarray(at, at + HEX_BYTES).toString('hex') + f.x
        at += HEX_BYTES
        return id
      })
    } else {
      ids = sizes.map(() => text[line++])
    }
    return {
      name: f.n,
      poster: f.p,
      subject: f.j,
      postedAt: f.d,
      groups: f.g,
      segments: sizes.map((bytes, i) => [f.k?.[i] ?? i + 1, ids[i], bytes] as [number, string, number]),
    }
  })
}

function isNyuu(id: string, suffix: string): boolean {
  if (id.length !== 38 + suffix.length || id.charCodeAt(24) !== 45 || !id.endsWith(suffix)) return false
  for (let i = 0; i < 24; i += 2) {
    const upper = id.charCodeAt(i)
    const lower = id.charCodeAt(i + 1)
    if (upper < 65 || upper > 90 || lower < 97 || lower > 122) return false
  }
  for (let i = 25; i < 38; i++) {
    const c = id.charCodeAt(i)
    if (c < 48 || c > 57) return false
  }
  return true
}

function isHex(id: string, suffix: string): boolean {
  if (id.length !== 32 + suffix.length || !id.endsWith(suffix)) return false
  for (let i = 0; i < 32; i++) {
    const c = id.charCodeAt(i)
    if (!((c >= 48 && c <= 57) || (c >= 97 && c <= 102))) return false
  }
  return true
}

/** 12 capital+small letter pairs (checked by isNyuu), 10 bits a pair (26 × 26 < 1024): 120 bits in 15 bytes at `at`. */
function packLetters(letters: string, out: Buffer, at: number) {
  let acc = 0
  let bits = 0
  let o = at
  for (let i = 0; i < 24; i += 2) {
    acc = (acc << 10) | ((letters.charCodeAt(i) - 65) * 26 + (letters.charCodeAt(i + 1) - 97))
    bits += 10
    while (bits >= 8) {
      bits -= 8
      out[o++] = (acc >> bits) & 0xff
    }
    acc &= (1 << bits) - 1
  }
}

function unpackLetters(data: Buffer): string {
  let acc = 0
  let bits = 0
  let i = 0
  let out = ''
  for (let pair = 0; pair < 12; pair++) {
    while (bits < 10) {
      acc = (acc << 8) | data[i++]
      bits += 8
    }
    bits -= 10
    const v = (acc >> bits) & 0x3ff
    acc &= (1 << bits) - 1
    out += String.fromCharCode(65 + Math.floor(v / 26), 97 + (v % 26))
  }
  return out
}

/** Version 1 (2026-10-09, before the id codes): zstd of the JSON line and every message-id as text. */
function decodeV1(data: Buffer): ChunkFile[] {
  const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
  let at = 0
  return (JSON.parse(head) as any[]).map((f) => ({
    name: f.n,
    poster: f.p,
    subject: f.j,
    postedAt: f.d,
    groups: f.g,
    segments: (f.s as number[]).map((bytes, i) => [f.k?.[i] ?? i + 1, ids[at++], bytes] as [number, string, number]),
  }))
}