diff --git a/REBUILD.md b/REBUILD.md index 3c1face..6325b00 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -121,7 +121,9 @@ dropping garbage; 28 TB of compressed overview traffic. Throughput goal: cost: a second implementation of lib/headers, lib/release, lib/rules and the chunk format, kept identical (check.ts) for as long as both exist. Findings, how to run both and how to compare their outputs: - `tools/edge-bench-rs/README.md`. + `tools/edge-bench-rs/README.md`. Since 2026-10-09 the Rust worker + (`edge-node`, same crate) is the one running on homelab: ~8× less CPU + per header and 20× less memory than `edge:node` in production. Per lease: compressed chunks of 10–50 k lines, in order → parse subjects → skip crossposts outside their owner group (of the indexed groups in the article's Xref, the one with the lowest priority diff --git a/config/coordinator.ts b/config/coordinator.ts index 0928e4c..7faa61a 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -30,8 +30,13 @@ export default { leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60), /** NNTP connections the coordinator fetches headers with. */ nntpConnections: env.get('NNTP_CONNECTIONS', 4), - /** A reply with no data for this long fails (old ranges answer slowly). */ - nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 10 * 60 * 1000), + /** + * A reply with no data for this long fails, and its connection is replaced. + * Eweka sometimes stops answering on long-lived connections (all 40 quiet, + * a fresh one answers in seconds; LOG.md 2026-10-09 (14)): at 10 minutes + * the prefetcher nearly stopped. Data arrives within seconds otherwise. + */ + nntpTimeoutMs: env.get('NNTP_TIMEOUT_MS', 2 * 60 * 1000), spool: { dir: env.get('SPOOL_DIR', app.tmpPath('spool')), /** Ranges kept fetched ahead, per active lease. */ diff --git a/tools/edge-bench-rs/README.md b/tools/edge-bench-rs/README.md index 5d4fc80..8ec00c5 100644 --- a/tools/edge-bench-rs/README.md +++ b/tools/edge-bench-rs/README.md @@ -1,5 +1,13 @@ # edge-bench-rs: the edge worker's indexing in Rust +Since 2026-10-09 this crate is also the production worker: `edge-node` +(`src/bin/edge-node.rs`) leases ranges from the coordinator and posts +batches like `node ace edge:node`, with the same indexing as the +benchmark (`src/lib.rs`). On homelab it indexed 122M headers in 10 +minutes with 0.37 cores and 565 MB, next to the TypeScript workers' 24M +with 0.61 cores and 11.3 GB (8 threads vs 8 processes): about 8× less +CPU per header and 20× less memory. See "The worker" below. + A measurement, not a product. It answers whether a Rust edge worker would be worth a second implementation of the TypeScript one (`edge:node`, REBUILD.md part 2). It ports the path that `edge:bench` replays over a captured range: @@ -195,3 +203,32 @@ them: | `src/codec.rs` | lib/chunks encodeChunk, format v2 | | `src/js.rs` | JavaScript semantics, sha1 | | `check.ts` | the comparison | + +## The worker + +```sh +cargo build --release +COORDINATOR_URL=http://coordinator:3334 NODE_TOKEN=… ./target/release/edge-node [--threads N] [--batch-articles 500000] [--ranges N] [--max-mbit M] +``` + +- **One range per thread** (default one per core): one thread reads the + coordinator's stream and inflates a frame ahead, another posts the + batches in order (one may wait), the indexer goes on meanwhile. +- **As `commands/edge_node.ts`:** retries for network trouble and 5xx; + 409 means the lease is lost (the range stops, isn't given back); a + batch once 500k articles or 100k files wait (a picture group's batches + went over the server's 256 MB limit); a chunk that doesn't inflate is + reported (the coordinator fetches it plain next time); SIGTERM/SIGINT + give each range back after its current chunk; `--max-mbit` is the + node's download cap, shared by its threads. Plus a heartbeat when no + batch went out for a minute. +- **Wire format:** `src/wire.rs` writes a batch as `toWire` does. + `edge-bench-rs --wire` and `check.ts --wire` compare it with the + TypeScript's, field by field. +- **End to end:** both workers indexed the a.b.models capture (spooled + as a range) through the web server; what the server stored was the + same (releases with every column, chunks, candidates, rule counts, + fragments with their JSON). +- **On homelab:** container `usenet-rebuild-edge-rs` runs + `/mnt/bulk/usenet-rebuild/bin/edge-node --threads 8` in + node:22-bookworm-slim (the binary needs glibc 2.34+). diff --git a/tools/probe_chunk.ts b/tools/probe_chunk.ts new file mode 100644 index 0000000..a356498 --- /dev/null +++ b/tools/probe_chunk.ts @@ -0,0 +1,26 @@ +/** + * One XOVER chunk fetched compressed (with a reply timeout) and plain, timed: + * whether a chunk that never finishes compressed is whole without compression. + * + * node --env-file=.env --import=@poppinss/ts-exec tools/probe_chunk.ts [timeoutSeconds] + */ +import { unzipSync } from 'node:zlib' +import { XoverClient } from '#lib/nntp/index' + +const [group, from, to, timeout] = process.argv.slice(2) +const env = process.env +const connect = (compress: boolean, timeoutMs: number) => + XoverClient.connect({ host: env.NNTP_HOST!, port: Number(env.NNTP_PORT ?? 563), user: env.NNTP_USER, password: env.NNTP_PASSWORD, secure: (env.NNTP_SECURE ?? 'true') !== 'false', compress, timeoutMs }) +for (const compress of [true, false]) { + const c = await connect(compress, Number(timeout ?? 90) * 1000) + await c.group(group) + const at = Date.now() + try { + const raw = await c.xoverRaw(Number(from), Number(to)) + const lines = unzipSync(raw).toString().split('\r\n').filter((l) => l.includes('\t')).length + console.log(`${compress ? 'compressed' : 'plain'}: ${raw.length} bytes, ${lines} lines in ${((Date.now() - at) / 1000).toFixed(1)} s`) + } catch (e: any) { + console.log(`${compress ? 'compressed' : 'plain'}: ${e.message} after ${((Date.now() - at) / 1000).toFixed(1)} s`) + } + c.close() +}