Rebuild: chunk format v2 — nyuu and hex message-ids as bits

nyuu's 12 capital+small letter pairs in 15 bytes (10 bits a pair), its
timestamp as a delta within the file, the suffix once per file;
ngPost-style 32 hex digits in 16 bytes; other ids as zstd text as before.
Recent boneless: 25.2 → 20.3 B per segment (and 25.0 → 19.5 on the next
range); a 2013 ath range stays at 17.9 (JBinUp ids are text). Every chunk
of three real ranges decodes to itself (edge:bench --verify). Encoding is
no slower than v1 (less text for zstd). Version 1 chunks stay readable.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

commands/edge_bench.ts

@@ -7,6 +7,7 @@ import env from '#start/env'
 import { XoverClient } from '#lib/nntp/index'
 import { RangeIndexer } from '#lib/edge/index'
 import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
+import { decodeChunk, encodeChunk } from '#lib/chunks/index'
 import db from '@adonisjs/lucid/services/db'
 import BatchStore from '#services/batch_store'
 
@@ -45,6 +46,9 @@ export default class EdgeBench extends BaseCommand {
   @flags.number({ description: 'Chunks without a new part before an open file is given up', default: 5 })
   declare idle: number
 
+  @flags.boolean({ description: 'Decode every release chunk and encode it again: the same bytes' })
+  declare verify: boolean
+
   @flags.boolean({ description: 'Also store the batches (BatchStore, the configured database), timed' })
   declare store: boolean
 
@@ -137,6 +141,12 @@ export default class EdgeBench extends BaseCommand {
       out.releases += batch.releases.length
       batches++
       for (const r of batch.releases) {
+        if (this.verify) {
+          const decoded = decodeChunk(r.payload)
+          if (!encodeChunk(decoded).equals(r.payload) || decoded.reduce((n, f) => n + f.segments.length, 0) !== r.segments) {
+            throw new Error(`Release ${r.key}: its chunk doesn't decode to itself`)
+          }
+        }
         keys.add(r.key)
         out.files += r.files
         out.segments += r.segments

lib/chunks/codec.ts

@@ -11,27 +11,177 @@ export interface ChunkFile {
   segments: [number, string, number][]
 }
 
-const VERSION = 1
+/**
+ * Message-ids of the two tools that post most (LOG.md 2026-10-08 (3)):
+ * nyuu's 12 random capital+small letter pairs, a ms timestamp and a fixed
+ * suffix; ngPost-style 32 random hex digits and a fixed suffix. Their random
+ * parts are stored as bits (zstd can't shrink random text much), the rest
+ * as text.
+ */
+const NYUU_BYTES = 15
+const HEX_BYTES = 16
+
+const zstd = (data: Buffer) => zstdCompressSync(data, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })
 
 /**
- * A release chunk's payload (REBUILD.md, chunk store): a version byte, then
- * zstd of a JSON line with the files (name, poster, subject, date, groups,
- * sizes, and part numbers unless they are 1..N) followed by one message-id
- * per line, in file order. Message-ids are mostly random, so this is close
- * to what plain compression gets (LOG.md 2026-10-08 (3)).
+ * A release chunk's payload (REBUILD.md, chunk store), version 2: a version
+ * byte, the length of the compressed part (4 bytes), then zstd of a JSON
+ * line with the files (name, poster, subject, date, groups, sizes, part
+ * numbers unless 1..N, and per file how its ids are kept) and the other
+ * files' message-ids one per line, then the raw bytes of the nyuu and hex
+ * ids. ~25 → ~18 B per segment on boneless (LOG.md 2026-10-09).
  */
 export function encodeChunk(files: ChunkFile[]): Buffer {
-  const head = files.map((f) => {
-    const sequential = f.segments.every(([n], i) => n === i + 1)
-    return { n: f.name, p: f.poster, j: f.subject, d: f.postedAt, g: f.groups, s: f.segments.map(([, , b]) => b), ...(sequential ? {} : { k: f.segments.map(([n]) => n) }) }
+  const text: string[] = []
+  // First how each file's ids are kept, then all raw ids into one buffer (a buffer per id was slow).
+  const codes: ('n' | 'h' | 't')[] = files.map((f) => {
+    const ids = f.segments
+    const first = ids[0]?.[1]
+    if (!first) return 't'
+    const nyuuSuffix = first.slice(38)
+    if (nyuuSuffix.startsWith('@') && ids.every(([, id]) => isNyuu(id, nyuuSuffix))) return 'n'
+    const hexSuffix = first.slice(32)
+    if (hexSuffix.startsWith('@') && ids.every(([, id]) => isHex(id, hexSuffix))) return 'h'
+    return 't'
   })
-  const ids = files.flatMap((f) => f.segments.map(([, id]) => id))
-  const text = Buffer.from([JSON.stringify(head), ...ids].join('\n'), 'utf8')
-  return Buffer.concat([Buffer.from([VERSION]), zstdCompressSync(text, { params: { [constants.ZSTD_c_compressionLevel]: 3 } })])
+  let rawBytes = 0
+  files.forEach((f, i) => (rawBytes += codes[i] === 'n' ? NYUU_BYTES * f.segments.length : codes[i] === 'h' ? HEX_BYTES * f.segments.length : 0))
+  const raw = Buffer.allocUnsafe(rawBytes)
+  let at = 0
+  const head = files.map((f, i) => {
+    const sequential = f.segments.every(([n], k) => n === k + 1)
+    const meta: Record<string, unknown> = {
+      n: f.name,
+      p: f.poster,
+      j: f.subject,
+      d: f.postedAt,
+      g: f.groups,
+      s: f.segments.map(([, , b]) => b),
+      ...(sequential ? {} : { k: f.segments.map(([n]) => n) }),
+    }
+    if (codes[i] === 'n') {
+      let previous = 0
+      const deltas: number[] = []
+      for (const [, id] of f.segments) {
+        packLetters(id, raw, at)
+        at += NYUU_BYTES
+        const t = Number(id.slice(25, 38))
+        deltas.push(t - previous)
+        previous = t
+      }
+      return { ...meta, c: 'n', x: f.segments[0][1].slice(38), t: deltas }
+    }
+    if (codes[i] === 'h') {
+      for (const [, id] of f.segments) at += raw.write(id.slice(0, 32), at, 'hex')
+      return { ...meta, c: 'h', x: f.segments[0][1].slice(32) }
+    }
+    for (const [, id] of f.segments) text.push(id)
+    return meta
+  })
+  const packed = zstd(Buffer.from([JSON.stringify(head), ...text].join('\n'), 'utf8'))
+  const prefix = Buffer.alloc(5)
+  prefix[0] = 2
+  prefix.writeUInt32BE(packed.length, 1)
+  return Buffer.concat([prefix, packed, raw])
 }
 
 export function decodeChunk(data: Buffer): ChunkFile[] {
-  if (data[0] !== VERSION) throw new Error(`Unknown chunk version ${data[0]}`)
+  if (data[0] === 1) return decodeV1(data)
+  if (data[0] !== 2) throw new Error(`Unknown chunk version ${data[0]}`)
+  const length = data.readUInt32BE(1)
+  const [headLine, ...text] = zstdDecompressSync(data.subarray(5, 5 + length)).toString('utf8').split('\n')
+  let at = 5 + length
+  let line = 0
+  return (JSON.parse(headLine) as any[]).map((f) => {
+    const sizes = f.s as number[]
+    let ids: string[]
+    if (f.c === 'n') {
+      let t = 0
+      ids = sizes.map((_, i) => {
+        t += f.t[i]
+        const letters = unpackLetters(data.subarray(at, at + NYUU_BYTES))
+        at += NYUU_BYTES
+        return `${letters}-${String(t).padStart(13, '0')}${f.x}`
+      })
+    } else if (f.c === 'h') {
+      ids = sizes.map(() => {
+        const id = data.subarray(at, at + HEX_BYTES).toString('hex') + f.x
+        at += HEX_BYTES
+        return id
+      })
+    } else {
+      ids = sizes.map(() => text[line++])
+    }
+    return {
+      name: f.n,
+      poster: f.p,
+      subject: f.j,
+      postedAt: f.d,
+      groups: f.g,
+      segments: sizes.map((bytes, i) => [f.k?.[i] ?? i + 1, ids[i], bytes] as [number, string, number]),
+    }
+  })
+}
+
+function isNyuu(id: string, suffix: string): boolean {
+  if (id.length !== 38 + suffix.length || id.charCodeAt(24) !== 45 || !id.endsWith(suffix)) return false
+  for (let i = 0; i < 24; i += 2) {
+    const upper = id.charCodeAt(i)
+    const lower = id.charCodeAt(i + 1)
+    if (upper < 65 || upper > 90 || lower < 97 || lower > 122) return false
+  }
+  for (let i = 25; i < 38; i++) {
+    const c = id.charCodeAt(i)
+    if (c < 48 || c > 57) return false
+  }
+  return true
+}
+
+function isHex(id: string, suffix: string): boolean {
+  if (id.length !== 32 + suffix.length || !id.endsWith(suffix)) return false
+  for (let i = 0; i < 32; i++) {
+    const c = id.charCodeAt(i)
+    if (!((c >= 48 && c <= 57) || (c >= 97 && c <= 102))) return false
+  }
+  return true
+}
+
+/** 12 capital+small letter pairs (checked by isNyuu), 10 bits a pair (26 × 26 < 1024): 120 bits in 15 bytes at `at`. */
+function packLetters(letters: string, out: Buffer, at: number) {
+  let acc = 0
+  let bits = 0
+  let o = at
+  for (let i = 0; i < 24; i += 2) {
+    acc = (acc << 10) | ((letters.charCodeAt(i) - 65) * 26 + (letters.charCodeAt(i + 1) - 97))
+    bits += 10
+    while (bits >= 8) {
+      bits -= 8
+      out[o++] = (acc >> bits) & 0xff
+    }
+    acc &= (1 << bits) - 1
+  }
+}
+
+function unpackLetters(data: Buffer): string {
+  let acc = 0
+  let bits = 0
+  let i = 0
+  let out = ''
+  for (let pair = 0; pair < 12; pair++) {
+    while (bits < 10) {
+      acc = (acc << 8) | data[i++]
+      bits += 8
+    }
+    bits -= 10
+    const v = (acc >> bits) & 0x3ff
+    acc &= (1 << bits) - 1
+    out += String.fromCharCode(65 + Math.floor(v / 26), 97 + (v % 26))
+  }
+  return out
+}
+
+/** Version 1 (2026-10-09, before the id codes): zstd of the JSON line and every message-id as text. */
+function decodeV1(data: Buffer): ChunkFile[] {
   const [head, ...ids] = zstdDecompressSync(data.subarray(1)).toString('utf8').split('\n')
   let at = 0
   return (JSON.parse(head) as any[]).map((f) => ({

tests/unit/chunks/codec.spec.ts

@@ -8,10 +8,33 @@ test.group('chunks / codec', () => {
       { name: 'a.par2', poster: 'p <p@x>', subject: 'A [2/2] - "a.par2" yEnc', postedAt: null, groups: ['alt.binaries.test', 'alt.binaries.x'], segments: [[2, 'gap@x', 10], [5, 'gap2@x', 11]] },
     ]
     const data = encodeChunk(files)
-    assert.equal(data[0], 1)
+    assert.equal(data[0], 2)
     assert.deepEqual(decodeChunk(data), files)
   })
 
+  test('nyuu and hex ids as bits, others as text, all exactly back', ({ assert }) => {
+    const files: ChunkFile[] = [
+      { name: 'n.part1.rar', poster: 'p', subject: 'n', postedAt: 1, groups: ['a.b.x'],
+        segments: [[1, 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu', 716800], [2, 'ZzAaYyBbXxCcWwDdVvEeUuFf-1790637260490@nyuu', 716800], [3, 'AaAaAaAaAaAaAaAaAaAaAaAa-0790637260500@nyuu', 5]] },
+      { name: 'h.part1.rar', poster: 'p', subject: 'h', postedAt: 2, groups: ['a.b.x'],
+        segments: [[1, 'b94268bc702040f7b811d5318f727e4f@ngPost', 700000], [2, '00000000000000000000000000000000@ngPost', 1]] },
+      // Mixed suffixes: kept as text.
+      { name: 'm.bin', poster: 'p', subject: 'm', postedAt: null, groups: ['a.b.x'],
+        segments: [[1, 'b94268bc702040f7b811d5318f727e4f@ngPost', 1], [3, 'b94268bc702040f7b811d5318f727e4f@other', 2]] },
+      { name: 't.nfo', poster: 'p', subject: 't', postedAt: 3, groups: ['a.b.x'], segments: [[1, 'part1of1.abc@JBinUp.local', 9]] },
+    ]
+    const data = encodeChunk(files)
+    assert.equal(data[0], 2)
+    assert.deepEqual(decodeChunk(data), files)
+  })
+
+  test('reads version 1 chunks', async ({ assert }) => {
+    const { zstdCompressSync } = await import('node:zlib')
+    const head = JSON.stringify([{ n: 'a.rar', p: 'p', j: 'a', d: 5, g: ['a.b.x'], s: [10, 20] }])
+    const v1 = Buffer.concat([Buffer.from([1]), zstdCompressSync(Buffer.from([head, 'one@x', 'two@x'].join('\n')))])
+    assert.deepEqual(decodeChunk(v1), [{ name: 'a.rar', poster: 'p', subject: 'a', postedAt: 5, groups: ['a.b.x'], segments: [[1, 'one@x', 10], [2, 'two@x', 20]] }])
+  })
+
   test('refuses an unknown version', ({ assert }) => {
     assert.throws(() => decodeChunk(Buffer.from([9, 0])), /Unknown chunk version 9/)
   })