Idle connections fetch capture ranges into the corpus
The spool only fetches what workers lease soon, in priority order, and the tail never ends: after a reset the 1,400 capture ranges waited behind it while 34 of 40 connections idled. Up to CORPUS_AHEAD (8) planned capture ranges whose chunks aren't all stored are now targets after everything else, fetched into the corpus only (no spool); a later lease reads them from disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0fafe71b21
commit
7b436f3f77
5 files changed
+106
-3
No files matched your search
+19
-1
@@ -1,4 +1,4 @@
|
||||
import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { unzip } from 'node:zlib'
|
||||
import { promisify } from 'node:util'
|
||||
@@ -35,6 +35,24 @@ class Corpus {
|
||||
return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
}
|
||||
|
||||
/** Whether every chunk of the range is stored (its range.json). */
|
||||
async complete(range: CorpusRange): Promise<boolean> {
|
||||
if (!this.dir) return false
|
||||
return access(join(this.rangeDir(range), 'range.json')).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
}
|
||||
|
||||
/** Whether one chunk is stored. */
|
||||
async has(range: CorpusRange, span: ChunkSpan): Promise<boolean> {
|
||||
if (!this.dir) return false
|
||||
return access(this.path(range, span)).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
}
|
||||
|
||||
/** A stored chunk, or null. */
|
||||
async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
|
||||
if (!this.dir) return null
|
||||
|
||||
Reference in new issue
Block a user