From e9592dfd33df147a1d94e8660eb6eedb505cf4c4 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 02:31:59 +0200 Subject: [PATCH] =?UTF-8?q?Rebuild:=20chunk=20format=20v2=20=E2=80=94=20ny?= =?UTF-8?q?uu=20and=20hex=20message-ids=20as=20bits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nyuu's 12 capital+small letter pairs in 15 bytes (10 bits a pair), its timestamp as a delta within the file, the suffix once per file; ngPost-style 32 hex digits in 16 bytes; other ids as zstd text as before. Recent boneless: 25.2 → 20.3 B per segment (and 25.0 → 19.5 on the next range); a 2013 ath range stays at 17.9 (JBinUp ids are text). Every chunk of three real ranges decodes to itself (edge:bench --verify). Encoding is no slower than v1 (less text for zstd). Version 1 chunks stay readable. Co-Authored-By: Claude Opus 5.5 --- commands/edge_bench.ts | 10 ++ lib/chunks/codec.ts | 176 +++++++++++++++++++++++++++++--- tests/unit/chunks/codec.spec.ts | 25 ++++- 3 files changed, 197 insertions(+), 14 deletions(-) diff --git a/commands/edge_bench.ts b/commands/edge_bench.ts index e2db663..10e38ef 100644 --- a/commands/edge_bench.ts +++ b/commands/edge_bench.ts @@ -7,6 +7,7 @@ import env from '#start/env' import { XoverClient } from '#lib/nntp/index' import { RangeIndexer } from '#lib/edge/index' import { DEFAULT_RULES, type Rules } from '#lib/rules/index' +import { decodeChunk, encodeChunk } from '#lib/chunks/index' import db from '@adonisjs/lucid/services/db' import BatchStore from '#services/batch_store' @@ -45,6 +46,9 @@ export default class EdgeBench extends BaseCommand { @flags.number({ description: 'Chunks without a new part before an open file is given up', default: 5 }) declare idle: number + @flags.boolean({ description: 'Decode every release chunk and encode it again: the same bytes' }) + declare verify: boolean + @flags.boolean({ description: 'Also store the batches (BatchStore, the configured database), timed' }) declare store: boolean @@ -137,6 +141,12 @@ export default class EdgeBench extends BaseCommand { out.releases += batch.releases.length batches++ for (const r of batch.releases) { + if (this.verify) { + const decoded = decodeChunk(r.payload) + if (!encodeChunk(decoded).equals(r.payload) || decoded.reduce((n, f) => n + f.segments.length, 0) !== r.segments) { + throw new Error(`Release ${r.key}: its chunk doesn't decode to itself`) + } + } keys.add(r.key) out.files += r.files out.segments += r.segments diff --git a/lib/chunks/codec.ts b/lib/chunks/codec.ts index f51afdf..c0375c6 100644 --- a/lib/chunks/codec.ts +++ b/lib/chunks/codec.ts @@ -11,27 +11,177 @@ export interface ChunkFile { segments: [number, string, number][] } -const VERSION = 1 +/** + * Message-ids of the two tools that post most (LOG.md 2026-10-08 (3)): + * nyuu's 12 random capital+small letter pairs, a ms timestamp and a fixed + * suffix; ngPost-style 32 random hex digits and a fixed suffix. Their random + * parts are stored as bits (zstd can't shrink random text much), the rest + * as text. + */ +const NYUU_BYTES = 15 +const HEX_BYTES = 16 + +const zstd = (data: Buffer) => zstdCompressSync(data, { params: { [constants.ZSTD_c_compressionLevel]: 3 } }) /** - * A release chunk's payload (REBUILD.md, chunk store): a version byte, then - * zstd of a JSON line with the files (name, poster, subject, date, groups, - * sizes, and part numbers unless they are 1..N) followed by one message-id - * per line, in file order. Message-ids are mostly random, so this is close - * to what plain compression gets (LOG.md 2026-10-08 (3)). + * A release chunk's payload (REBUILD.md, chunk store), version 2: a version + * byte, the length of the compressed part (4 bytes), then zstd of a JSON + * line with the files (name, poster, subject, date, groups, sizes, part + * numbers unless 1..N, and per file how its ids are kept) and the other + * files' message-ids one per line, then the raw bytes of the nyuu and hex + * ids. ~25 → ~18 B per segment on boneless (LOG.md 2026-10-09). */ export function encodeChunk(files: ChunkFile[]): Buffer { - const head = files.map((f) => { - const sequential = f.segments.every(([n], i) => n === i + 1) - return { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }) } + const text: string[] = [] + // First how each file's ids are kept, then all raw ids into one buffer (a buffer per id was slow). + const codes: ('n' | 'h' | 't')[] = files.map((f) => { + const ids = f.segments + const first = ids[0]?.[1] + if (!first) return 't' + const nyuuSuffix = first.slice(38) + if (nyuuSuffix.startsWith('@') && ids.every(([, id]) => isNyuu(id, nyuuSuffix))) return 'n' + const hexSuffix = first.slice(32) + if (hexSuffix.startsWith('@') && ids.every(([, id]) => isHex(id, hexSuffix))) return 'h' + return 't' }) - const ids = files.flatMap((f) => f.segments.map(([, id]) => id)) - const text = Buffer.from([JSON.stringify(head), ...ids].join('\n'), 'utf8') - return Buffer.concat([Buffer.from([VERSION]), zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })]) + let rawBytes = 0 + files.forEach((f, i) => (rawBytes += codes[i] === 'n' ? NYUU_BYTES * f.segments.length : codes[i] === 'h' ? HEX_BYTES * f.segments.length : 0)) + const raw = Buffer.allocUnsafe(rawBytes) + let at = 0 + const head = files.map((f, i) => { + const sequential = f.segments.every(([n], k) => n === k + 1) + const meta: Record = { + n: f.name, + p: f.poster, + j: f.subject, + d: f.postedAt, + g: f.groups, + s: f.segments.map(([, , b]) => b), + ...(sequential ? {} : { k: f.segments.map(([n]) => n) }), + } + if (codes[i] === 'n') { + let previous = 0 + const deltas: number[] = [] + for (const [, id] of f.segments) { + packLetters(id, raw, at) + at += NYUU_BYTES + const t = Number(id.slice(25, 38)) + deltas.push(t - previous) + previous = t + } + return { ...meta, c: 'n', x: f.segments[0][1].slice(38), t: deltas } + } + if (codes[i] === 'h') { + for (const [, id] of f.segments) at += raw.write(id.slice(0, 32), at, 'hex') + return { ...meta, c: 'h', x: f.segments[0][1].slice(32) } + } + for (const [, id] of f.segments) text.push(id) + return meta + }) + const packed = zstd(Buffer.from([JSON.stringify(head), ...text].join('\n'), 'utf8')) + const prefix = Buffer.alloc(5) + prefix[0] = 2 + prefix.writeUInt32BE(packed.length, 1) + return Buffer.concat([prefix, packed, raw]) } export function decodeChunk(data: Buffer): ChunkFile[] { - if (data[0] !== VERSION) throw new Error(`Unknown chunk version ${data[0]}`) + if (data[0] === 1) return decodeV1(data) + if (data[0] !== 2) throw new Error(`Unknown chunk version ${data[0]}`) + const length = data.readUInt32BE(1) + const [headLine, ...text] = zstdDecompressSync(data.subarray(5, 5 + length)).toString('utf8').split('\n') + let at = 5 + length + let line = 0 + return (JSON.parse(headLine) as any[]).map((f) => { + const sizes = f.s as number[] + let ids: string[] + if (f.c === 'n') { + let t = 0 + ids = sizes.map((_, i) => { + t += f.t[i] + const letters = unpackLetters(data.subarray(at, at + NYUU_BYTES)) + at += NYUU_BYTES + return `${letters}-${String(t).padStart(13, '0')}${f.x}` + }) + } else if (f.c === 'h') { + ids = sizes.map(() => { + const id = data.subarray(at, at + HEX_BYTES).toString('hex') + f.x + at += HEX_BYTES + return id + }) + } else { + ids = sizes.map(() => text[line++]) + } + return { + name: f.n, + poster: f.p, + subject: f.j, + postedAt: f.d, + groups: f.g, + segments: sizes.map((bytes, i) => [f.k?.[i] ?? i + 1, ids[i], bytes] as [number, string, number]), + } + }) +} + +function isNyuu(id: string, suffix: string): boolean { + if (id.length !== 38 + suffix.length || id.charCodeAt(24) !== 45 || !id.endsWith(suffix)) return false + for (let i = 0; i < 24; i += 2) { + const upper = id.charCodeAt(i) + const lower = id.charCodeAt(i + 1) + if (upper < 65 || upper > 90 || lower < 97 || lower > 122) return false + } + for (let i = 25; i < 38; i++) { + const c = id.charCodeAt(i) + if (c < 48 || c > 57) return false + } + return true +} + +function isHex(id: string, suffix: string): boolean { + if (id.length !== 32 + suffix.length || !id.endsWith(suffix)) return false + for (let i = 0; i < 32; i++) { + const c = id.charCodeAt(i) + if (!((c >= 48 && c <= 57) || (c >= 97 && c <= 102))) return false + } + return true +} + +/** 12 capital+small letter pairs (checked by isNyuu), 10 bits a pair (26 × 26 < 1024): 120 bits in 15 bytes at `at`. */ +function packLetters(letters: string, out: Buffer, at: number) { + let acc = 0 + let bits = 0 + let o = at + for (let i = 0; i < 24; i += 2) { + acc = (acc << 10) | ((letters.charCodeAt(i) - 65) * 26 + (letters.charCodeAt(i + 1) - 97)) + bits += 10 + while (bits >= 8) { + bits -= 8 + out[o++] = (acc >> bits) & 0xff + } + acc &= (1 << bits) - 1 + } +} + +function unpackLetters(data: Buffer): string { + let acc = 0 + let bits = 0 + let i = 0 + let out = '' + for (let pair = 0; pair < 12; pair++) { + while (bits < 10) { + acc = (acc << 8) | data[i++] + bits += 8 + } + bits -= 10 + const v = (acc >> bits) & 0x3ff + acc &= (1 << bits) - 1 + out += String.fromCharCode(65 + Math.floor(v / 26), 97 + (v % 26)) + } + return out +} + +/** Version 1 (2026-10-09, before the id codes): zstd of the JSON line and every message-id as text. */ +function decodeV1(data: Buffer): ChunkFile[] { const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n') let at = 0 return (JSON.parse(head) as any[]).map((f) => ({ diff --git a/tests/unit/chunks/codec.spec.ts b/tests/unit/chunks/codec.spec.ts index 624bc03..8ad1db7 100644 --- a/tests/unit/chunks/codec.spec.ts +++ b/tests/unit/chunks/codec.spec.ts @@ -8,10 +8,33 @@ test.group('chunks / codec', () => { { name: 'a.par2', poster: 'p ', subject: 'A [2/2] - "a.par2" yEnc', postedAt: null, groups: ['alt.binaries.test', 'alt.binaries.x'], segments: [[2, 'gap@x', 10], [5, 'gap2@x', 11]] }, ] const data = encodeChunk(files) - assert.equal(data[0], 1) + assert.equal(data[0], 2) assert.deepEqual(decodeChunk(data), files) }) + test('nyuu and hex ids as bits, others as text, all exactly back', ({ assert }) => { + const files: ChunkFile[] = [ + { name: 'n.part1.rar', poster: 'p', subject: 'n', postedAt: 1, groups: ['a.b.x'], + segments: [[1, 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu', 716800], [2, 'ZzAaYyBbXxCcWwDdVvEeUuFf-1790637260490@nyuu', 716800], [3, 'AaAaAaAaAaAaAaAaAaAaAaAa-0790637260500@nyuu', 5]] }, + { name: 'h.part1.rar', poster: 'p', subject: 'h', postedAt: 2, groups: ['a.b.x'], + segments: [[1, 'b94268bc702040f7b811d5318f727e4f@ngPost', 700000], [2, '00000000000000000000000000000000@ngPost', 1]] }, + // Mixed suffixes: kept as text. + { name: 'm.bin', poster: 'p', subject: 'm', postedAt: null, groups: ['a.b.x'], + segments: [[1, 'b94268bc702040f7b811d5318f727e4f@ngPost', 1], [3, 'b94268bc702040f7b811d5318f727e4f@other', 2]] }, + { name: 't.nfo', poster: 'p', subject: 't', postedAt: 3, groups: ['a.b.x'], segments: [[1, 'part1of1.abc@JBinUp.local', 9]] }, + ] + const data = encodeChunk(files) + assert.equal(data[0], 2) + assert.deepEqual(decodeChunk(data), files) + }) + + test('reads version 1 chunks', async ({ assert }) => { + const { zstdCompressSync } = await import('node:zlib') + const head = JSON.stringify([{ n: 'a.rar', p: 'p', j: 'a', d: 5, g: ['a.b.x'], s: [10, 20] }]) + const v1 = Buffer.concat([Buffer.from([1]), zstdCompressSync(Buffer.from([head, 'one@x', 'two@x'].join('\n')))]) + assert.deepEqual(decodeChunk(v1), [{ name: 'a.rar', poster: 'p', subject: 'a', postedAt: 5, groups: ['a.b.x'], segments: [[1, 'one@x', 10], [2, 'two@x', 20]] }]) + }) + test('refuses an unknown version', ({ assert }) => { assert.throws(() => decodeChunk(Buffer.from([9, 0])), /Unknown chunk version 9/) })