The pending ladder in the prober: unnamed releases probed cheapest first, signatures learned, the named moved into the catalog
pending_ladder.ts claims the oldest waiting pending releases' next target (par2, archive, lone, 7z-tail in that order) as at least PENDING_SHARE of each claim; a usable name (the largest named file of a par2 set or a listing, or a lone file's yEnc name, neither packaging nor random) hands the release to the namer, no name the next target or the end of the climb; climbs count on pending_signatures per (tool, group, poster) and (tool, group), and a signature with 100 climbs and 0.5% named or fewer is dropped: its waiting releases are discarded without a probe, the rules' sample excepted. pending_namer.ts reads a named release's articles again from the corpus or by XOVER and writes it to the catalog under the pending key (source weak-stem, the name as inner name) with its chunk and candidates. Migration 6400; config PENDING_*. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
9cf85e09d8
commit
e135076fcd
14 files changed
+1045
-50
No files matched your search
@@ -42,13 +42,13 @@ class Corpus {
|
||||
return coordinatorConfig.corpus.dir
|
||||
}
|
||||
|
||||
private rangeDir(range: CorpusRange) {
|
||||
return join(this.dir!, range.group, `${range.start}-${range.end}`)
|
||||
private rangeDir(range: CorpusRange, dir = this.dir!) {
|
||||
return join(dir, range.group, `${range.start}-${range.end}`)
|
||||
}
|
||||
|
||||
private path(range: CorpusRange, span: ChunkSpan) {
|
||||
private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
|
||||
const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
|
||||
return join(this.rangeDir(range), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
}
|
||||
|
||||
/** Whether every chunk of the range is stored (its range.json). */
|
||||
@@ -69,10 +69,10 @@ class Corpus {
|
||||
)
|
||||
}
|
||||
|
||||
/** A stored chunk, or null. */
|
||||
async read(range: CorpusRange, span: ChunkSpan): Promise<Buffer | null> {
|
||||
if (!this.dir) return null
|
||||
return readFile(this.path(range, span)).catch(() => null)
|
||||
/** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
|
||||
async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
|
||||
if (!dir) return null
|
||||
return readFile(this.path(range, span, dir)).catch(() => null)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user