Rebuild: phase 1 modules from the current app, ordered by pipeline stage
lib/nntp (XOVER client, Xref, GROUP), lib/headers (subjects, file keys, segments from overview lines; the yEnc path stays behind for phase 2), lib/release (identity, random names, now in lib), lib/files (aggregator, volume names), lib/rules (posting tools, owner by priority, release verdicts), lib/chunks (release chunk codec). Their unit tests came along; new ones for codec, rules and volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
96f0ade10e
commit
d773c91ad0
26 files changed
+1486
No files matched your search
@@ -0,0 +1,45 @@
|
||||
import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib'
|
||||
|
||||
/** A file of a release chunk: what an NZB needs, once per file, and its segments as [part, message-id, bytes]. */
|
||||
export interface ChunkFile {
|
||||
name: string
|
||||
poster: string
|
||||
subject: string
|
||||
/** Earliest post date of its articles, ms since 1970 (null when no article had a valid date). */
|
||||
postedAt: number | null
|
||||
groups: string[]
|
||||
segments: [number, string, number][]
|
||||
}
|
||||
|
||||
const VERSION = 1
|
||||
|
||||
/**
|
||||
* A release chunk's payload (REBUILD.md, chunk store): a version byte, then
|
||||
* zstd of a JSON line with the files (name, poster, subject, date, groups,
|
||||
* sizes, and part numbers unless they are 1..N) followed by one message-id
|
||||
* per line, in file order. Message-ids are mostly random, so this is close
|
||||
* to what plain compression gets (LOG.md 2026-10-08 (3)).
|
||||
*/
|
||||
export function encodeChunk(files: ChunkFile[]): Buffer {
|
||||
const head = files.map((f) => {
|
||||
const sequential = f.segments.every(([n], i) => n === i + 1)
|
||||
return { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }) }
|
||||
})
|
||||
const ids = files.flatMap((f) => f.segments.map(([, id]) => id))
|
||||
const text = Buffer.from([JSON.stringify(head), ...ids].join('\n'), 'utf8')
|
||||
return Buffer.concat([Buffer.from([VERSION]), zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })])
|
||||
}
|
||||
|
||||
export function decodeChunk(data: Buffer): ChunkFile[] {
|
||||
if (data[0] !== VERSION) throw new Error(`Unknown chunk version ${data[0]}`)
|
||||
const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
|
||||
let at = 0
|
||||
return (JSON.parse(head) as any[]).map((f) => ({
|
||||
name: f.n,
|
||||
poster: f.p,
|
||||
subject: f.j,
|
||||
postedAt: f.d,
|
||||
groups: f.g,
|
||||
segments: (f.s as number[]).map((bytes, i) => [f.k?.[i] ?? i + 1, ids[at++], bytes] as [number, string, number]),
|
||||
}))
|
||||
}
|
||||
Reference in new issue
Block a user