Rebuild: a lean overview scan for the edge worker
The worker reads the XOVER reply's text directly: fields by position, the owner from the Xref without arrays (crossposts leave before any subject parsing), a subject parsed once per file (cached without its segment counter), a file's details as one shared object, dates as numbers. Garbage collection had been ~22% of the time. Same results as parseOverviews + segmentFromHeader on 800k real lines. edge:bench (one core): boneless 137k → 193k headers/s (CPU 21 → 12.8 s), ath (2013) 143k → 303k; peak RSS 1.15 / 0.87 GB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
981bc01810
commit
638cfa1dcd
5 files changed
+292
-59
No files matched your search
@@ -2,7 +2,6 @@ import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { gzipSync, unzipSync } from 'node:zlib'
|
||||
import { parseOverviews } from '#lib/nntp/index'
|
||||
import { RangeIndexer, decodeFrames, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
|
||||
|
||||
const IDLE_MS = 30_000
|
||||
@@ -81,7 +80,7 @@ export default class EdgeNode extends BaseCommand {
|
||||
}
|
||||
try {
|
||||
for await (const frame of decodeFrames(await api.stream(range.id))) {
|
||||
indexer.add(parseOverviews(unzipSync(frame.data).toString('utf8')), frame.to + 1)
|
||||
indexer.addText(unzipSync(frame.data).toString('utf8'), frame.to + 1)
|
||||
chunks++
|
||||
const last = frame.to >= range.end
|
||||
if (last) indexer.finish()
|
||||
|
||||
Reference in new issue
Block a user