Rebuild: 2-minute NNTP reply timeout; the Rust worker runs on homelab
Eweka stopped answering on all 40 of the coordinator's long-lived connections (quiet sockets; a fresh connection fetched the same chunk in 20 s): with a 10-minute timeout the prefetcher fetched one chunk a minute. The timeout (silence, not duration) is 2 minutes now: 365-411k articles/s after the restart. tools/probe_chunk.ts times a chunk compressed and plain. README and REBUILD.md: edge-node, the Rust worker, is the one running (8x less CPU per header, 20x less memory than edge:node in production). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a8a139c6d9
commit
8442c4fc44
4 files changed
+73
-3
No files matched your search
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* One XOVER chunk fetched compressed (with a reply timeout) and plain, timed:
|
||||
* whether a chunk that never finishes compressed is whole without compression.
|
||||
*
|
||||
* node --env-file=.env --import=@poppinss/ts-exec tools/probe_chunk.ts <group> <from> <to> [timeoutSeconds]
|
||||
*/
|
||||
import { unzipSync } from 'node:zlib'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
|
||||
const [group, from, to, timeout] = process.argv.slice(2)
|
||||
const env = process.env
|
||||
const connect = (compress: boolean, timeoutMs: number) =>
|
||||
XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress, timeoutMs })
|
||||
for (const compress of [true, false]) {
|
||||
const c = await connect(compress, Number(timeout ?? 90) * 1000)
|
||||
await c.group(group)
|
||||
const at = Date.now()
|
||||
try {
|
||||
const raw = await c.xoverRaw(Number(from), Number(to))
|
||||
const lines = unzipSync(raw).toString().split('\r\n').filter((l) => l.includes('\t')).length
|
||||
console.log(`${compress ? 'compressed' : 'plain'}: ${raw.length} bytes, ${lines} lines in ${((Date.now() - at) / 1000).toFixed(1)} s`)
|
||||
} catch (e: any) {
|
||||
console.log(`${compress ? 'compressed' : 'plain'}: ${e.message} after ${((Date.now() - at) / 1000).toFixed(1)} s`)
|
||||
}
|
||||
c.close()
|
||||
}
|
||||
Reference in new issue
Block a user