Rebuild: a lean overview scan for the edge worker
Test and deploy / test (push) Successful in 22s
Test and deploy / deploy (push) Skipped

The worker reads the XOVER reply's text directly: fields by position, the
owner from the Xref without arrays (crossposts leave before any subject
parsing), a subject parsed once per file (cached without its segment
counter), a file's details as one shared object, dates as numbers.
Garbage collection had been ~22% of the time.

Same results as parseOverviews + segmentFromHeader on 800k real lines.
edge:bench (one core): boneless 137k → 193k headers/s (CPU 21 → 12.8 s),
ath (2013) 143k → 303k; peak RSS 1.15 / 0.87 GB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 01:13:41 +02:00
1 parent 981bc01810
commit 638cfa1dcd
5 files changed
+292 -59

No files matched your search

+3 -5
View File
@@ -4,7 +4,7 @@ import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { unzipSync } from 'node:zlib'
import env from '#start/env'
import { XoverClient, parseOverviews } from '#lib/nntp/index'
import { XoverClient } from '#lib/nntp/index'
import { RangeIndexer } from '#lib/edge/index'
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
import db from '@adonisjs/lucid/services/db'
@@ -117,11 +117,9 @@ export default class EdgeBench extends BaseCommand {
t.inflate += performance.now() - at
text += data.length
at = performance.now()
const overviews = parseOverviews(data)
t.parse += performance.now() - at
at = performance.now()
const next = Number(file.split('-')[2].replace('.gz', '')) + 1
indexer.add(overviews, next)
// The worker's path: the reply's text straight into the indexer (parsing is part of "index").
indexer.addText(data, next)
const last = file === files.at(-1)
if (last) indexer.finish()
t.index += performance.now() - at
+1 -2
View File
@@ -2,7 +2,6 @@ import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import { setTimeout as sleep } from 'node:timers/promises'
import { gzipSync, unzipSync } from 'node:zlib'
import { parseOverviews } from '#lib/nntp/index'
import { RangeIndexer, decodeFrames, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
const IDLE_MS = 30_000
@@ -81,7 +80,7 @@ export default class EdgeNode extends BaseCommand {
}
try {
for await (const frame of decodeFrames(await api.stream(range.id))) {
indexer.add(parseOverviews(unzipSync(frame.data).toString('utf8')), frame.to + 1)
indexer.addText(unzipSync(frame.data).toString('utf8'), frame.to + 1)
chunks++
const last = frame.to >= range.end
if (last) indexer.finish()