Daan MeijerandClaude Opus 5.5
d773c91ad0
lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys, segments from overview lines; the yEnc path stays behind for phase 2), lib/release (identity, random names, now in lib), lib/files (aggregator, volume names), lib/rules (posting tools, owner by priority, release verdicts), lib/chunks (release chunk codec). Their unit tests came along; new ones for codec, rules and volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
46 lines
1.9 KiB
TypeScript
46 lines
1.9 KiB
TypeScript
import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib'
|
|
|
|
/** A file of a release chunk: what an NZB needs, once per file, and its segments as [part, message-id, bytes]. */
|
|
export interface ChunkFile {
|
|
name: string
|
|
poster: string
|
|
subject: string
|
|
/** Earliest post date of its articles, ms since 1970 (null when no article had a valid date). */
|
|
postedAt: number | null
|
|
groups: string[]
|
|
segments: [number, string, number][]
|
|
}
|
|
|
|
const VERSION = 1
|
|
|
|
/**
|
|
* A release chunk's payload (REBUILD.md, chunk store): a version byte, then
|
|
* zstd of a JSON line with the files (name, poster, subject, date, groups,
|
|
* sizes, and part numbers unless they are 1..N) followed by one message-id
|
|
* per line, in file order. Message-ids are mostly random, so this is close
|
|
* to what plain compression gets (LOG.md 2026-10-08 (3)).
|
|
*/
|
|
export function encodeChunk(files: ChunkFile[]): Buffer {
|
|
const head = files.map((f) => {
|
|
const sequential = f.segments.every(([n], i) => n === i + 1)
|
|
return { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }) }
|
|
})
|
|
const ids = files.flatMap((f) => f.segments.map(([, id]) => id))
|
|
const text = Buffer.from([JSON.stringify(head), ...ids].join('\n'), 'utf8')
|
|
return Buffer.concat([Buffer.from([VERSION]), zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })])
|
|
}
|
|
|
|
export function decodeChunk(data: Buffer): ChunkFile[] {
|
|
if (data[0] !== VERSION) throw new Error(`Unknown chunk version ${data[0]}`)
|
|
const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
|
|
let at = 0
|
|
return (JSON.parse(head) as any[]).map((f) => ({
|
|
name: f.n,
|
|
poster: f.p,
|
|
subject: f.j,
|
|
postedAt: f.d,
|
|
groups: f.g,
|
|
segments: (f.s as number[]).map((bytes, i) => [f.k?.[i] ?? i + 1, ids[at++], bytes] as [number, string, number]),
|
|
}))
|
|
}
|