Daan MeijerandClaude Opus 5.5
7b436f3f77
The spool only fetches what workers lease soon, in priority order, and the tail never ends: after a reset the 1,400 capture ranges waited behind it while 34 of 40 connections idled. Up to CORPUS_AHEAD (8) planned capture ranges whose chunks aren't all stored are now targets after everything else, fetched into the corpus only (no spool); a later lease reads them from disk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
91 lines
3.0 KiB
TypeScript
91 lines
3.0 KiB
TypeScript
import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
|
import { join } from 'node:path'
|
|
import { unzip } from 'node:zlib'
|
|
import { promisify } from 'node:util'
|
|
import coordinatorConfig from '#config/coordinator'
|
|
import type { ChunkSpan } from '#services/spool'
|
|
|
|
const inflate = promisify(unzip)
|
|
|
|
export interface CorpusRange {
|
|
group: string
|
|
start: number
|
|
end: number
|
|
}
|
|
|
|
/**
|
|
* Stored XOVER chunks of the ranges marked `capture` (REBUILD.md, corpus):
|
|
* `<dir>/<group>/<start>-<end>/<nnnn>-<from>-<to>.gz` with a range.json once
|
|
* every chunk is there, the layout edge-bench reads (edge/fixtures). The
|
|
* grouping sweeps replay them with each build's rules instead of waiting for
|
|
* a re-index; the prefetcher reads a stored chunk instead of fetching it
|
|
* again. Off without CORPUS_DIR.
|
|
*/
|
|
class Corpus {
|
|
get dir(): string | undefined {
|
|
return coordinatorConfig.corpus.dir
|
|
}
|
|
|
|
private rangeDir(range: CorpusRange) {
|
|
return join(this.dir!, range.group, `${range.start}-${range.end}`)
|
|
}
|
|
|
|
private path(range: CorpusRange, span: ChunkSpan) {
|
|
const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
|
|
return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
|
}
|
|
|
|
/** Whether every chunk of the range is stored (its range.json). */
|
|
async complete(range: CorpusRange): Promise<boolean> {
|
|
if (!this.dir) return false
|
|
return access(join(this.rangeDir(range), 'range.json')).then(
|
|
() => true,
|
|
() => false
|
|
)
|
|
}
|
|
|
|
/** Whether one chunk is stored. */
|
|
async has(range: CorpusRange, span: ChunkSpan): Promise<boolean> {
|
|
if (!this.dir) return false
|
|
return access(this.path(range, span)).then(
|
|
() => true,
|
|
() => false
|
|
)
|
|
}
|
|
|
|
/** A stored chunk, or null. */
|
|
async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
|
|
if (!this.dir) return null
|
|
return readFile(this.path(range, span)).catch(() => null)
|
|
}
|
|
|
|
/**
|
|
* Stores a chunk that inflates (a reply cut short isn't kept), and the
|
|
* range.json once the range is complete (`total` chunks).
|
|
*/
|
|
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number) {
|
|
if (!this.dir) return
|
|
try {
|
|
await inflate(data)
|
|
} catch {
|
|
return
|
|
}
|
|
const dir = this.rangeDir(range)
|
|
const file = this.path(range, span)
|
|
await mkdir(dir, { recursive: true })
|
|
await writeFile(`${file}.part`, data)
|
|
await rename(`${file}.part`, file)
|
|
const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
|
|
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(range))
|
|
}
|
|
|
|
/** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
|
|
async removeChunk(range: CorpusRange, span: ChunkSpan) {
|
|
if (!this.dir) return
|
|
await rm(this.path(range, span), { force: true })
|
|
await rm(join(this.rangeDir(range), 'range.json'), { force: true })
|
|
}
|
|
}
|
|
|
|
export default new Corpus()
|