Daan MeijerandClaude Opus 5.5
fc1c64d5c0
A file name with a NUL ("…Brutal Erpresst.ra\u0000…") failed every
batch of range 155595 (HTTP 500, jsonb: \u0000 cannot be converted to
text), so the range looped. fromWire drops NUL from every string.
The chunk deadline was given up 258 times in 3 hours at 300 s (~1.4 a
minute, each holding a connection for 5 minutes): 180 s.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
37 lines
1.5 KiB
TypeScript
37 lines
1.5 KiB
TypeScript
import type { EdgeBatch } from './range_indexer.js'
|
|
|
|
/** An EdgeBatch as JSON: chunk payloads in base64. */
|
|
export type WireBatch = Omit<EdgeBatch, 'releases' | 'samples'> & {
|
|
releases: (Omit<EdgeBatch['releases'][number], 'payload'> & { payload: string })[]
|
|
samples: (Omit<EdgeBatch['samples'][number], 'payload'> & { payload: string })[]
|
|
}
|
|
|
|
export function toWire(batch: EdgeBatch): WireBatch {
|
|
return {
|
|
...batch,
|
|
releases: batch.releases.map((r) => ({ ...r, payload: r.payload.toString('base64') })),
|
|
samples: batch.samples.map((s) => ({ ...s, payload: s.payload.toString('base64') })),
|
|
}
|
|
}
|
|
|
|
/** Strings without NUL: Postgres takes it in neither text nor jsonb, and a file name with one failed every batch of its range (LOG.md 2026-10-10 (3)). */
|
|
function withoutNul<T>(value: T): T {
|
|
if (typeof value === 'string') return (value.includes('\u0000') ? value.replaceAll('\u0000', '') : value) as T
|
|
if (Array.isArray(value)) return value.map(withoutNul) as T
|
|
if (value && typeof value === 'object') {
|
|
const out: Record<string, unknown> = {}
|
|
for (const [k, v] of Object.entries(value)) out[k] = withoutNul(v)
|
|
return out as T
|
|
}
|
|
return value
|
|
}
|
|
|
|
export function fromWire(raw: WireBatch): EdgeBatch {
|
|
const wire = withoutNul(raw)
|
|
return {
|
|
...wire,
|
|
releases: wire.releases.map((r) => ({ ...r, payload: Buffer.from(r.payload, 'base64') })),
|
|
samples: wire.samples.map((s) => ({ ...s, payload: Buffer.from(s.payload, 'base64') })),
|
|
}
|
|
}
|