Rebuild: chunk format v2 — nyuu and hex message-ids as bits
nyuu's 12 capital+small letter pairs in 15 bytes (10 bits a pair), its timestamp as a delta within the file, the suffix once per file; ngPost-style 32 hex digits in 16 bytes; other ids as zstd text as before. Recent boneless: 25.2 → 20.3 B per segment (and 25.0 → 19.5 on the next range); a 2013 ath range stays at 17.9 (JBinUp ids are text). Every chunk of three real ranges decodes to itself (edge:bench --verify). Encoding is no slower than v1 (less text for zstd). Version 1 chunks stay readable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
31493a98be
commit
e9592dfd33
3 files changed
+197
-14
No files matched your search
@@ -7,6 +7,7 @@ import env from '#start/env'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { RangeIndexer } from '#lib/edge/index'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
import { decodeChunk, encodeChunk } from '#lib/chunks/index'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import BatchStore from '#services/batch_store'
|
||||
|
||||
@@ -45,6 +46,9 @@ export default class EdgeBench extends BaseCommand {
|
||||
@flags.number({ description: 'Chunks without a new part before an open file is given up', default: 5 })
|
||||
declare idle: number
|
||||
|
||||
@flags.boolean({ description: 'Decode every release chunk and encode it again: the same bytes' })
|
||||
declare verify: boolean
|
||||
|
||||
@flags.boolean({ description: 'Also store the batches (BatchStore, the configured database), timed' })
|
||||
declare store: boolean
|
||||
|
||||
@@ -137,6 +141,12 @@ export default class EdgeBench extends BaseCommand {
|
||||
out.releases += batch.releases.length
|
||||
batches++
|
||||
for (const r of batch.releases) {
|
||||
if (this.verify) {
|
||||
const decoded = decodeChunk(r.payload)
|
||||
if (!encodeChunk(decoded).equals(r.payload) || decoded.reduce((n, f) => n + f.segments.length, 0) !== r.segments) {
|
||||
throw new Error(`Release ${r.key}: its chunk doesn't decode to itself`)
|
||||
}
|
||||
}
|
||||
keys.add(r.key)
|
||||
out.files += r.files
|
||||
out.segments += r.segments
|
||||
|
||||
Reference in new issue
Block a user