usenet-indexer/edge/src/codec.rs

//! lib/chunks/codec.ts encodeChunk, format v2: a version byte, the length of
//! the compressed part, zstd(JSON head + text ids), then the nyuu and hex ids
//! as raw bytes.

use crate::js::json_str;

pub struct ChunkFile<'a> {
    pub name: &'a str,
    pub poster: &'a str,
    pub subject: &'a str,
    pub posted_at: Option<i64>,
    pub groups: &'a [String],
    /// [part, message-id, bytes]
    pub segments: Vec<(u32, &'a str, i64)>,
}

fn is_nyuu(id: &str, suffix: &str) -> bool {
    let b = id.as_bytes();
    if b.len() != 38 + suffix.len() || b[24] != b'-' || !id.ends_with(suffix) {
        return false;
    }
    (0..24).step_by(2).all(|i| b[i].is_ascii_uppercase() && b[i + 1].is_ascii_lowercase()) && b[25..38].iter().all(u8::is_ascii_digit)
}

fn is_hex(id: &str, suffix: &str) -> bool {
    let b = id.as_bytes();
    b.len() == 32 + suffix.len() && id.ends_with(suffix) && b[..32].iter().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(c))
}

fn pack_letters(id: &[u8], out: &mut Vec<u8>) {
    let mut acc: u32 = 0;
    let mut bits = 0;
    for i in (0..24).step_by(2) {
        acc = (acc << 10) | ((id[i] - b'A') as u32 * 26 + (id[i + 1] - b'a') as u32);
        bits += 10;
        while bits >= 8 {
            bits -= 8;
            out.push(((acc >> bits) & 0xff) as u8);
        }
        acc &= (1 << bits) - 1;
    }
}

fn hex_val(c: u8) -> u8 {
    if c.is_ascii_digit() {
        c - b'0'
    } else {
        c - b'a' + 10
    }
}

fn push_num(out: &mut String, n: i64) {
    use std::fmt::Write;
    write!(out, "{n}").unwrap();
}

pub fn encode_chunk(files: &[ChunkFile]) -> Vec<u8> {
    let mut head = String::with_capacity(256 * files.len());
    let mut text: Vec<&str> = Vec::new();
    let mut raw: Vec<u8> = Vec::new();
    head.push('[');
    for (fi, f) in files.iter().enumerate() {
        let ids = &f.segments;
        let code = match ids.first() {
            None => 't',
            Some(&(_, first, _)) => {
                let nyuu = first.get(38..).filter(|s| s.starts_with('@'));
                let hex = first.get(32..).filter(|s| s.starts_with('@'));
                if nyuu.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_nyuu(id, suffix))) {
                    'n'
                } else if hex.is_some_and(|suffix| ids.iter().all(|&(_, id, _)| is_hex(id, suffix))) {
                    'h'
                } else {
                    't'
                }
            }
        };
        if fi > 0 {
            head.push(',');
        }
        head.push_str("{\"n\":");
        json_str(f.name, &mut head);
        head.push_str(",\"p\":");
        json_str(f.poster, &mut head);
        head.push_str(",\"j\":");
        json_str(f.subject, &mut head);
        head.push_str(",\"d\":");
        match f.posted_at {
            Some(t) => push_num(&mut head, t),
            None => head.push_str("null"),
        }
        head.push_str(",\"g\":[");
        for (i, g) in f.groups.iter().enumerate() {
            if i > 0 {
                head.push(',');
            }
            json_str(g, &mut head);
        }
        head.push_str("],\"s\":[");
        for (i, &(_, _, b)) in ids.iter().enumerate() {
            if i > 0 {
                head.push(',');
            }
            push_num(&mut head, b);
        }
        head.push(']');
        if !ids.iter().enumerate().all(|(k, &(n, _, _))| n as usize == k + 1) {
            head.push_str(",\"k\":[");
            for (i, &(n, _, _)) in ids.iter().enumerate() {
                if i > 0 {
                    head.push(',');
                }
                push_num(&mut head, n as i64);
            }
            head.push(']');
        }
        match code {
            'n' => {
                head.push_str(",\"c\":\"n\",\"x\":");
                json_str(&ids[0].1[38..], &mut head);
                head.push_str(",\"t\":[");
                let mut previous = 0i64;
                for (i, &(_, id, _)) in ids.iter().enumerate() {
                    pack_letters(id.as_bytes(), &mut raw);
                    let t: i64 = id[25..38].parse().unwrap();
                    if i > 0 {
                        head.push(',');
                    }
                    push_num(&mut head, t - previous);
                    previous = t;
                }
                head.push(']');
            }
            'h' => {
                head.push_str(",\"c\":\"h\",\"x\":");
                json_str(&ids[0].1[32..], &mut head);
                for &(_, id, _) in ids {
                    let b = id.as_bytes();
                    for i in (0..32).step_by(2) {
                        raw.push(hex_val(b[i]) << 4 | hex_val(b[i + 1]));
                    }
                }
            }
            _ => text.extend(ids.iter().map(|&(_, id, _)| id)),
        }
        head.push('}');
    }
    head.push(']');
    let mut plain = head.into_bytes();
    for id in text {
        plain.push(b'\n');
        plain.extend_from_slice(id.as_bytes());
    }
    let packed = zstd::bulk::compress(&plain, 3).unwrap();
    let mut out = Vec::with_capacity(5 + packed.len() + raw.len());
    out.push(2);
    out.extend_from_slice(&(packed.len() as u32).to_be_bytes());
    out.extend_from_slice(&packed);
    out.extend_from_slice(&raw);
    out
}