Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
- groups:add / ranges:plan / nodes:create / coordinator:run commands - the node API: lease, the range's chunks as one framed stream from the spool, batches, heartbeat, finish, release (node guard, token per node) - edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for the rest), posts a batch every 10 chunks, gives its range back on SIGTERM - releases are gathered per batch: 1.04 chunk rows per release at 10 chunks per batch (1.35 when built per XOVER chunk) - partitions are checked per batch instead of remembered Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
20910647fa
commit
d5f168d8d2
36 files changed
+1193
-14
No files matched your search
@@ -0,0 +1,80 @@
|
||||
import env from '#start/env'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
|
||||
/**
|
||||
* The coordinator's header connections (REBUILD.md, decision e: workers
|
||||
* have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
|
||||
* that failed is closed and replaced on the next use.
|
||||
*/
|
||||
class NntpPool {
|
||||
private idle: XoverClient[] = []
|
||||
private open = 0
|
||||
private waiting: ((client: XoverClient) => void)[] = []
|
||||
|
||||
get size() {
|
||||
return coordinatorConfig.nntpConnections
|
||||
}
|
||||
|
||||
async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
const client = await this.acquire()
|
||||
let broken = false
|
||||
try {
|
||||
return await work(client)
|
||||
} catch (error) {
|
||||
broken = true
|
||||
throw error
|
||||
} finally {
|
||||
this.release(client, broken)
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<XoverClient> {
|
||||
const ready = this.idle.pop()
|
||||
if (ready) return ready
|
||||
if (this.open < this.size) {
|
||||
this.open++
|
||||
try {
|
||||
return await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: true,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
} catch (error) {
|
||||
this.open--
|
||||
throw error
|
||||
}
|
||||
}
|
||||
return new Promise((resolve) => this.waiting.push(resolve))
|
||||
}
|
||||
|
||||
private release(client: XoverClient, broken: boolean) {
|
||||
if (broken) {
|
||||
this.open--
|
||||
try {
|
||||
client.close()
|
||||
} catch (error: any) {
|
||||
logger.debug(`Closing a failed NNTP connection: ${error.message}`)
|
||||
}
|
||||
// Someone waiting gets a fresh connection.
|
||||
const next = this.waiting.shift()
|
||||
if (next) this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
return
|
||||
}
|
||||
const next = this.waiting.shift()
|
||||
if (next) next(client)
|
||||
else this.idle.push(client)
|
||||
}
|
||||
|
||||
async close() {
|
||||
for (const client of this.idle.splice(0)) client.close()
|
||||
this.open = 0
|
||||
}
|
||||
}
|
||||
|
||||
export default new NntpPool()
|
||||
Reference in new issue
Block a user