//! lib/chunks/codec.ts encodeChunk, format v2: a version byte, the length of //! the compressed part, zstd(JSON head + text ids), then the nyuu and hex ids //! as raw bytes. use crate::js::json_str; pub struct ChunkFile<'a> { pub name: &'a str, pub poster: &'a str, pub subject: &'a str, pub posted_at: Option, pub groups: &'a [String], /// [part, message-id, bytes] pub segments: Vec<(u32, &'a str, i64)>, } fn is_nyuu(id: &str, suffix: &str) -> bool { let b = id.as_bytes(); if b.len() != 38 + suffix.len() || b[24] != b'-' || !id.ends_with(suffix) { return false; } (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[25..38].iter().all(u8::is_ascii_digit) } fn is_hex(id: &str, suffix: &str) -> bool { let b = id.as_bytes(); b.len() == 32 + suffix.len() && id.ends_with(suffix) && b[..32].iter().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(c)) } fn pack_letters(id: &[u8], out: &mut Vec) { let mut acc: u32 = 0; let mut bits = 0; for i in (0..24).step_by(2) { acc = (acc << 10) | ((id[i] - b'A') as u32 * 26 + (id[i + 1] - b'a') as u32); bits += 10; while bits >= 8 { bits -= 8; out.push(((acc >> bits) & 0xff) as u8); } acc &= (1 << bits) - 1; } } fn hex_val(c: u8) -> u8 { if c.is_ascii_digit() { c - b'0' } else { c - b'a' + 10 } } fn push_num(out: &mut String, n: i64) { use std::fmt::Write; write!(out, "{n}").unwrap(); } pub fn encode_chunk(files: &[ChunkFile]) -> Vec { let mut head = String::with_capacity(256 * files.len()); let mut text: Vec<&str> = Vec::new(); let mut raw: Vec = Vec::new(); head.push('['); for (fi, f) in files.iter().enumerate() { let ids = &f.segments; let code = match ids.first() { None => 't', Some(&(_, first, _)) => { let nyuu = first.get(38..).filter(|s| s.starts_with('@')); let hex = first.get(32..).filter(|s| s.starts_with('@')); if nyuu.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_nyuu(id, suffix))) { 'n' } else if hex.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_hex(id, suffix))) { 'h' } else { 't' } } }; if fi > 0 { head.push(','); } head.push_str("{\"n\":"); json_str(f.name, &mut head); head.push_str(",\"p\":"); json_str(f.poster, &mut head); head.push_str(",\"j\":"); json_str(f.subject, &mut head); head.push_str(",\"d\":"); match f.posted_at { Some(t) => push_num(&mut head, t), None => head.push_str("null"), } head.push_str(",\"g\":["); for (i, g) in f.groups.iter().enumerate() { if i > 0 { head.push(','); } json_str(g, &mut head); } head.push_str("],\"s\":["); for (i, &(_, _, b)) in ids.iter().enumerate() { if i > 0 { head.push(','); } push_num(&mut head, b); } head.push(']'); if !ids.iter().enumerate().all(|(k, &(n, _, _))| n as usize == k + 1) { head.push_str(",\"k\":["); for (i, &(n, _, _)) in ids.iter().enumerate() { if i > 0 { head.push(','); } push_num(&mut head, n as i64); } head.push(']'); } match code { 'n' => { head.push_str(",\"c\":\"n\",\"x\":"); json_str(&ids[0].1[38..], &mut head); head.push_str(",\"t\":["); let mut previous = 0i64; for (i, &(_, id, _)) in ids.iter().enumerate() { pack_letters(id.as_bytes(), &mut raw); let t: i64 = id[25..38].parse().unwrap(); if i > 0 { head.push(','); } push_num(&mut head, t - previous); previous = t; } head.push(']'); } 'h' => { head.push_str(",\"c\":\"h\",\"x\":"); json_str(&ids[0].1[32..], &mut head); for &(_, id, _) in ids { let b = id.as_bytes(); for i in (0..32).step_by(2) { raw.push(hex_val(b[i]) << 4 | hex_val(b[i + 1])); } } } _ => text.extend(ids.iter().map(|&(_, id, _)| id)), } head.push('}'); } head.push(']'); let mut plain = head.into_bytes(); for id in text { plain.push(b'\n'); plain.extend_from_slice(id.as_bytes()); } let packed = zstd::bulk::compress(&plain, 3).unwrap(); let mut out = Vec::with_capacity(5 + packed.len() + raw.len()); out.push(2); out.extend_from_slice(&(packed.len() as u32).to_be_bytes()); out.extend_from_slice(&packed); out.extend_from_slice(&raw); out }