Rebuild: drop NUL from batches; chunk deadline 180 s
A file name with a NUL ("…Brutal Erpresst.ra\u0000…") failed every
batch of range 155595 (HTTP 500, jsonb: \u0000 cannot be converted to
text), so the range looped. fromWire drops NUL from every string.
The chunk deadline was given up 258 times in 3 hours at 300 s (~1.4 a
minute, each holding a connection for 5 minutes): 180 s.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f4c0417066
commit
fc1c64d5c0
3 files changed
+39
-3
No files matched your search
@@ -41,9 +41,10 @@ export default {
|
||||
* A chunk still coming in after this long is given up (its connection
|
||||
* closed) and fetched again: a reply that trickles never goes quiet long
|
||||
* enough for nntpTimeoutMs, and kept a worker waiting on one chunk for an
|
||||
* hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s.
|
||||
* hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s; at 300 s, 258 were
|
||||
* given up in 3 hours (~1.4 a minute, each holding a connection 5 minutes).
|
||||
*/
|
||||
nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 5 * 60) * 1000,
|
||||
nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 3 * 60) * 1000,
|
||||
/**
|
||||
* A chunk not fetched after this long is asked again on another
|
||||
* connection; the first answer wins and the other connection is closed.
|
||||
|
||||
+14
-1
@@ -14,7 +14,20 @@ export function toWire(batch: EdgeBatch): WireBatch {
|
||||
}
|
||||
}
|
||||
|
||||
export function fromWire(wire: WireBatch): EdgeBatch {
|
||||
/** Strings without NUL: Postgres takes it in neither text nor jsonb, and a file name with one failed every batch of its range (LOG.md 2026-10-10 (3)). */
|
||||
function withoutNul<T>(value: T): T {
|
||||
if (typeof value === 'string') return (value.includes('\u0000') ? value.replaceAll('\u0000', '') : value) as T
|
||||
if (Array.isArray(value)) return value.map(withoutNul) as T
|
||||
if (value && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = withoutNul(v)
|
||||
return out as T
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function fromWire(raw: WireBatch): EdgeBatch {
|
||||
const wire = withoutNul(raw)
|
||||
return {
|
||||
...wire,
|
||||
releases: wire.releases.map((r) => ({ ...r, payload: Buffer.from(r.payload, 'base64') })),
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { test } from '@japa/runner'
|
||||
import { fromWire, type WireBatch } from '#lib/edge/index'
|
||||
|
||||
test.group('edge / wire', () => {
|
||||
test('fromWire drops NUL from strings, wherever they are', ({ assert }) => {
|
||||
const wire = {
|
||||
from: 1,
|
||||
next: 2,
|
||||
headers: 1,
|
||||
releases: [{ key: 'k', name: 'Brutal.ra\u0000r', payload: Buffer.from('x').toString('base64') }],
|
||||
samples: [],
|
||||
candidates: [],
|
||||
counts: [],
|
||||
fragments: [{ fileKey: 'f', segments: [['a\u0000b.rar', 1, 2, 3]] }],
|
||||
} as unknown as WireBatch
|
||||
const batch = fromWire(wire) as any
|
||||
assert.equal(batch.releases[0].name, 'Brutal.rar')
|
||||
assert.equal(batch.releases[0].payload.toString(), 'x')
|
||||
assert.equal(batch.fragments[0].segments[0][0], 'ab.rar')
|
||||
assert.equal(batch.fragments[0].segments[0][3], 3)
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user