diff --git a/config/coordinator.ts b/config/coordinator.ts index 2be09ed..c0c9ebf 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -41,9 +41,10 @@ export default { * A chunk still coming in after this long is given up (its connection * closed) and fetched again: a reply that trickles never goes quiet long * enough for nntpTimeoutMs, and kept a worker waiting on one chunk for an - * hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s. + * hour (LOG.md 2026-10-10 (2)). Chunks take 3–35 s; at 300 s, 258 were + * given up in 3 hours (~1.4 a minute, each holding a connection 5 minutes). */ - nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 5 * 60) * 1000, + nntpChunkDeadlineMs: env.get('NNTP_CHUNK_DEADLINE_SECONDS', 3 * 60) * 1000, /** * A chunk not fetched after this long is asked again on another * connection; the first answer wins and the other connection is closed. diff --git a/lib/edge/wire.ts b/lib/edge/wire.ts index b6f0652..54ea5cd 100644 --- a/lib/edge/wire.ts +++ b/lib/edge/wire.ts @@ -14,7 +14,20 @@ export function toWire(batch: EdgeBatch): WireBatch { } } -export function fromWire(wire: WireBatch): EdgeBatch { +/** Strings without NUL: Postgres takes it in neither text nor jsonb, and a file name with one failed every batch of its range (LOG.md 2026-10-10 (3)). */ +function withoutNul(value: T): T { + if (typeof value === 'string') return (value.includes('\u0000') ? value.replaceAll('\u0000', '') : value) as T + if (Array.isArray(value)) return value.map(withoutNul) as T + if (value && typeof value === 'object') { + const out: Record = {} + for (const [k, v] of Object.entries(value)) out[k] = withoutNul(v) + return out as T + } + return value +} + +export function fromWire(raw: WireBatch): EdgeBatch { + const wire = withoutNul(raw) return { ...wire, releases: wire.releases.map((r) => ({ ...r, payload: Buffer.from(r.payload, 'base64') })), diff --git a/tests/unit/edge/wire.spec.ts b/tests/unit/edge/wire.spec.ts new file mode 100644 index 0000000..af51205 --- /dev/null +++ b/tests/unit/edge/wire.spec.ts @@ -0,0 +1,22 @@ +import { test } from '@japa/runner' +import { fromWire, type WireBatch } from '#lib/edge/index' + +test.group('edge / wire', () => { + test('fromWire drops NUL from strings, wherever they are', ({ assert }) => { + const wire = { + from: 1, + next: 2, + headers: 1, + releases: [{ key: 'k', name: 'Brutal.ra\u0000r', payload: Buffer.from('x').toString('base64') }], + samples: [], + candidates: [], + counts: [], + fragments: [{ fileKey: 'f', segments: [['a\u0000b.rar', 1, 2, 3]] }], + } as unknown as WireBatch + const batch = fromWire(wire) as any + assert.equal(batch.releases[0].name, 'Brutal.rar') + assert.equal(batch.releases[0].payload.toString(), 'x') + assert.equal(batch.fragments[0].segments[0][0], 'ab.rar') + assert.equal(batch.fragments[0].segments[0][3], 3) + }) +})