Rebuild: drop NUL from batches; chunk deadline 180 s
Test and deploy / test (push) Successful in 24s
Test and deploy / deploy (push) Successful in 1m0s

A file name with a NUL ("…Brutal Erpresst.ra\u0000…") failed every
batch of range 155595 (HTTP 500, jsonb: \u0000 cannot be converted to
text), so the range looped. fromWire drops NUL from every string.

The chunk deadline was given up 258 times in 3 hours at 300 s (~1.4 a
minute, each holding a connection for 5 minutes): 180 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-10 13:12:27 +02:00
1 parent f4c0417066
commit fc1c64d5c0
3 files changed
+39 -3

No files matched your search

+3 -2
View File
@@ -41,9 +41,10 @@ export default {
* A chunk still coming in after this long is given up (its connection
* closed) and fetched again: a reply that trickles never goes quiet long
* enough for nntpTimeoutMs, and kept a worker waiting on one chunk for an
* hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s.
* hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s; at 300 s, 258 were
* given up in 3 hours (~1.4 a minute, each holding a connection 5 minutes).
*/
nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 5 * 60) * 1000,
nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 3 * 60) * 1000,
/**
* A chunk not fetched after this long is asked again on another
* connection; the first answer wins and the other connection is closed.
+14 -1
View File
@@ -14,7 +14,20 @@ export function toWire(batch: EdgeBatch): WireBatch {
}
}
export function fromWire(wire: WireBatch): EdgeBatch {
/** Strings without NUL: Postgres takes it in neither text nor jsonb, and a file name with one failed every batch of its range (LOG.md 2026-10-10 (3)). */
function withoutNul<T>(value: T): T {
if (typeof value === 'string') return (value.includes('\u0000') ? value.replaceAll('\u0000', '') : value) as T
if (Array.isArray(value)) return value.map(withoutNul) as T
if (value && typeof value === 'object') {
const out: Record<string, unknown> = {}
for (const [k, v] of Object.entries(value)) out[k] = withoutNul(v)
return out as T
}
return value
}
export function fromWire(raw: WireBatch): EdgeBatch {
const wire = withoutNul(raw)
return {
...wire,
releases: wire.releases.map((r) => ({ ...r, payload: Buffer.from(r.payload, 'base64') })),
+22
View File
@@ -0,0 +1,22 @@
import { test } from '@japa/runner'
import { fromWire, type WireBatch } from '#lib/edge/index'
test.group('edge / wire', () => {
test('fromWire drops NUL from strings, wherever they are', ({ assert }) => {
const wire = {
from: 1,
next: 2,
headers: 1,
releases: [{ key: 'k', name: 'Brutal.ra\u0000r', payload: Buffer.from('x').toString('base64') }],
samples: [],
candidates: [],
counts: [],
fragments: [{ fileKey: 'f', segments: [['a\u0000b.rar', 1, 2, 3]] }],
} as unknown as WireBatch
const batch = fromWire(wire) as any
assert.equal(batch.releases[0].name, 'Brutal.rar')
assert.equal(batch.releases[0].payload.toString(), 'x')
assert.equal(batch.fragments[0].segments[0][0], 'ab.rar')
assert.equal(batch.fragments[0].segments[0][3], 3)
})
})