import { zstdCompressSync, zstdDecompressSync, constants } from 'node:zlib' /** One article of a posted file: its number, message-id (without <>) and size in bytes. */ export interface Segment { number: number id: string size: number } const VERSION = 1 /** * Segment lists as stored in `files.segments` (CONCEPT.md D31): a version * byte, then zstd (level 1) of a JSON line with the sizes (and the numbers, * unless they are 1..N) followed by one message-id per line. About 20 bytes * per segment, against ~38 for the pglz-compressed JSON object it replaces: * message-ids are mostly random, so this is close to what's possible. */ export function packSegments(segments: Segment[]): Buffer { const sorted = [...segments].sort((a, b) => a.number - b.number) const sequential = sorted.every((s, i) => s.number === i + 1) const head = JSON.stringify(sequential ? { s: sorted.map((s) => s.size) } : { s: sorted.map((s) => s.size), n: sorted.map((s) => s.number) }) const text = Buffer.from([head, ...sorted.map((s) => s.id)].join('\n'), 'utf8') const packed = zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 1 } }) return Buffer.concat([Buffer.from([VERSION]), packed]) } export function unpackSegments(data: Buffer): Segment[] { if (data[0] !== VERSION) throw new Error(`Unknown segment list version ${data[0]}`) const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n') const { s: sizes, n: numbers } = JSON.parse(head) as { s: number[]; n?: number[] } return sizes.map((size, i) => ({ number: numbers?.[i] ?? i + 1, id: ids[i], size })) }