import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib' /** A file of a release chunk: what an NZB needs, once per file, and its segments as [part, message-id, bytes]. */ export interface ChunkFile { name: string poster: string subject: string /** Earliest post date of its articles, ms since 1970 (null when no article had a valid date). */ postedAt: number | null groups: string[] segments: [number, string, number][] } /** * Message-ids of the two tools that post most (LOG.md 2026-10-08 (3)): * nyuu's 12 random capital+small letter pairs, a ms timestamp and a fixed * suffix; ngPost-style 32 random hex digits and a fixed suffix. Their random * parts are stored as bits (zstd can't shrink random text much), the rest * as text. */ const NYUU_BYTES = 15 const HEX_BYTES = 16 const zstd = (data: Buffer) => zstdCompressSync(data, { params: { [constants.ZSTD_c_compressionLevel]: 3 } }) /** * A release chunk's payload (REBUILD.md, chunk store), version 2: a version * byte, the length of the compressed part (4 bytes), then zstd of a JSON * line with the files (name, poster, subject, date, groups, sizes, part * numbers unless 1..N, and per file how its ids are kept) and the other * files' message-ids one per line, then the raw bytes of the nyuu and hex * ids. ~25 → ~18 B per segment on boneless (LOG.md 2026-10-09). */ export function encodeChunk(files: ChunkFile[]): Buffer { const text: string[] = [] // First how each file's ids are kept, then all raw ids into one buffer (a buffer per id was slow). const codes: ('n' | 'h' | 't')[] = files.map((f) => { const ids = f.segments const first = ids[0]?.[1] if (!first) return 't' const nyuuSuffix = first.slice(38) if (nyuuSuffix.startsWith('@') && ids.every(([, id]) => isNyuu(id, nyuuSuffix))) return 'n' const hexSuffix = first.slice(32) if (hexSuffix.startsWith('@') && ids.every(([, id]) => isHex(id, hexSuffix))) return 'h' return 't' }) let rawBytes = 0 files.forEach((f, i) => (rawBytes += codes[i] === 'n' ? NYUU_BYTES * f.segments.length : codes[i] === 'h' ? HEX_BYTES * f.segments.length : 0)) const raw = Buffer.allocUnsafe(rawBytes) let at = 0 const head = files.map((f, i) => { const sequential = f.segments.every(([n], k) => n === k + 1) const meta: Record = { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }), } if (codes[i] === 'n') { let previous = 0 const deltas: number[] = [] for (const [, id] of f.segments) { packLetters(id, raw, at) at += NYUU_BYTES const t = Number(id.slice(25, 38)) deltas.push(t - previous) previous = t } return { ...meta, c: 'n', x: f.segments[0][1].slice(38), t: deltas } } if (codes[i] === 'h') { for (const [, id] of f.segments) at += raw.write(id.slice(0, 32), at, 'hex') return { ...meta, c: 'h', x: f.segments[0][1].slice(32) } } for (const [, id] of f.segments) text.push(id) return meta }) const packed = zstd(Buffer.from([JSON.stringify(head), ...text].join('\n'), 'utf8')) const prefix = Buffer.alloc(5) prefix[0] = 2 prefix.writeUInt32BE(packed.length, 1) return Buffer.concat([prefix, packed, raw]) } export function decodeChunk(data: Buffer): ChunkFile[] { if (data[0] === 1) return decodeV1(data) if (data[0] !== 2) throw new Error(`Unknown chunk version ${data[0]}`) const length = data.readUInt32BE(1) const [headLine, ...text] = zstdDecompressSync(data.subarray(5, 5 + length)).toString('utf8').split('\n') let at = 5 + length let line = 0 return (JSON.parse(headLine) as any[]).map((f) => { const sizes = f.s as number[] let ids: string[] if (f.c === 'n') { let t = 0 ids = sizes.map((_, i) => { t += f.t[i] const letters = unpackLetters(data.subarray(at, at + NYUU_BYTES)) at += NYUU_BYTES return `${letters}-${String(t).padStart(13, '0')}${f.x}` }) } else if (f.c === 'h') { ids = sizes.map(() => { const id = data.subarray(at, at + HEX_BYTES).toString('hex') + f.x at += HEX_BYTES return id }) } else { ids = sizes.map(() => text[line++]) } return { name: f.n, poster: f.p, subject: f.j, postedAt: f.d, groups: f.g, segments: sizes.map((bytes, i) => [f.k?.[i] ?? i + 1, ids[i], bytes] as [number, string, number]), } }) } function isNyuu(id: string, suffix: string): boolean { if (id.length !== 38 + suffix.length || id.charCodeAt(24) !== 45 || !id.endsWith(suffix)) return false for (let i = 0; i < 24; i += 2) { const upper = id.charCodeAt(i) const lower = id.charCodeAt(i + 1) if (upper < 65 || upper > 90 || lower < 97 || lower > 122) return false } for (let i = 25; i < 38; i++) { const c = id.charCodeAt(i) if (c < 48 || c > 57) return false } return true } function isHex(id: string, suffix: string): boolean { if (id.length !== 32 + suffix.length || !id.endsWith(suffix)) return false for (let i = 0; i < 32; i++) { const c = id.charCodeAt(i) if (!((c >= 48 && c <= 57) || (c >= 97 && c <= 102))) return false } return true } /** 12 capital+small letter pairs (checked by isNyuu), 10 bits a pair (26 × 26 < 1024): 120 bits in 15 bytes at `at`. */ function packLetters(letters: string, out: Buffer, at: number) { let acc = 0 let bits = 0 let o = at for (let i = 0; i < 24; i += 2) { acc = (acc << 10) | ((letters.charCodeAt(i) - 65) * 26 + (letters.charCodeAt(i + 1) - 97)) bits += 10 while (bits >= 8) { bits -= 8 out[o++] = (acc >> bits) & 0xff } acc &= (1 << bits) - 1 } } function unpackLetters(data: Buffer): string { let acc = 0 let bits = 0 let i = 0 let out = '' for (let pair = 0; pair < 12; pair++) { while (bits < 10) { acc = (acc << 8) | data[i++] bits += 8 } bits -= 10 const v = (acc >> bits) & 0x3ff acc &= (1 << bits) - 1 out += String.fromCharCode(65 + Math.floor(v / 26), 97 + (v % 26)) } return out } /** Version 1 (2026-10-09, before the id codes): zstd of the JSON line and every message-id as text. */ function decodeV1(data: Buffer): ChunkFile[] { const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n') let at = 0 return (JSON.parse(head) as any[]).map((f) => ({ name: f.n, poster: f.p, subject: f.j, postedAt: f.d, groups: f.g, segments: (f.s as number[]).map((bytes, i) => [f.k?.[i] ?? i + 1, ids[at++], bytes] as [number, string, number]), })) }