daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild: coordinator (planner, NNTP pool, spool, prefetcher, leases, rule sets), node API and the edge:node worker
Test and deploy / test (push) Successful in 21s
Details
Test and deploy / deploy (push) Skipped
Details

Browse Source 
- groups:add / ranges:plan / nodes:create / coordinator:run commands
- the node API: lease, the range's chunks as one framed stream from the
  spool, batches, heartbeat, finish, release (node guard, token per node)
- edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for
  the rest), posts a batch every 10 chunks, gives its range back on SIGTERM
- releases are gathered per batch: 1.04 chunk rows per release at 10
  chunks per batch (1.35 when built per XOVER chunk)
- partitions are checked per batch instead of remembered

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-09 00:31:06 +02:00
1 parent 20910647fa
commit d5f168d8d2
36 files changed
+1193 -14
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.adonisjs/client/manifest.d.ts
Vendored
+1
View File
@@ -5,5 +5,6 @@
/// <reference path="../../adonisrc.ts" />
/// <reference path="../../config/auth.ts" />
/// <reference path="../../config/coordinator.ts" />
/// <reference path="../../config/hash.ts" />
/// <reference path="../../config/logger.ts" />
.adonisjs/client/registry/index.ts
+36
View File
@@ -30,6 +30,42 @@ const routes = {
tokens: [{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],
types: placeholder as Registry['profile.access_tokens.destroy']['types'],
},
'node.node.lease': {
methods: ["POST"],
pattern: '/api/v1/node/lease',
tokens: [{"old":"/api/v1/node/lease","type":0,"val":"api","end":""},{"old":"/api/v1/node/lease","type":0,"val":"v1","end":""},{"old":"/api/v1/node/lease","type":0,"val":"node","end":""},{"old":"/api/v1/node/lease","type":0,"val":"lease","end":""}],
types: placeholder as Registry['node.node.lease']['types'],
},
'node.node.stream': {
methods: ["GET","HEAD"],
pattern: '/api/v1/node/ranges/:id/stream',
tokens: [{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"stream","end":""}],
types: placeholder as Registry['node.node.stream']['types'],
},
'node.node.batch': {
methods: ["POST"],
pattern: '/api/v1/node/ranges/:id/batches',
tokens: [{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"batches","end":""}],
types: placeholder as Registry['node.node.batch']['types'],
},
'node.node.heartbeat': {
methods: ["POST"],
pattern: '/api/v1/node/ranges/:id/heartbeat',
tokens: [{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"heartbeat","end":""}],
types: placeholder as Registry['node.node.heartbeat']['types'],
},
'node.node.finish': {
methods: ["POST"],
pattern: '/api/v1/node/ranges/:id/finish',
tokens: [{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"finish","end":""}],
types: placeholder as Registry['node.node.finish']['types'],
},
'node.node.release': {
methods: ["POST"],
pattern: '/api/v1/node/ranges/:id/release',
tokens: [{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/release","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"release","end":""}],
types: placeholder as Registry['node.node.release']['types'],
},
} as const satisfies Record<string, AdonisEndpoint>
export { routes }
.adonisjs/client/registry/schema.d.ts
Vendored
+72
View File
@@ -55,4 +55,76 @@ export interface Registry {
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['destroy']>>>
}
}
'node.node.lease': {
methods: ["POST"]
pattern: '/api/v1/node/lease'
types: {
body: {}
paramsTuple: []
params: {}
query: {}
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['lease']>>>
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['lease']>>>
}
}
'node.node.stream': {
methods: ["GET","HEAD"]
pattern: '/api/v1/node/ranges/:id/stream'
types: {
body: {}
paramsTuple: [ParamValue]
params: { id: ParamValue }
query: {}
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['stream']>>>
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['stream']>>>
}
}
'node.node.batch': {
methods: ["POST"]
pattern: '/api/v1/node/ranges/:id/batches'
types: {
body: {}
paramsTuple: [ParamValue]
params: { id: ParamValue }
query: {}
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['batch']>>>
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['batch']>>>
}
}
'node.node.heartbeat': {
methods: ["POST"]
pattern: '/api/v1/node/ranges/:id/heartbeat'
types: {
body: {}
paramsTuple: [ParamValue]
params: { id: ParamValue }
query: {}
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['heartbeat']>>>
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['heartbeat']>>>
}
}
'node.node.finish': {
methods: ["POST"]
pattern: '/api/v1/node/ranges/:id/finish'
types: {
body: {}
paramsTuple: [ParamValue]
params: { id: ParamValue }
query: {}
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['finish']>>>
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['finish']>>>
}
}
'node.node.release': {
methods: ["POST"]
pattern: '/api/v1/node/ranges/:id/release'
types: {
body: {}
paramsTuple: [ParamValue]
params: { id: ParamValue }
query: {}
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['release']>>>
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['release']>>>
}
}
}
.adonisjs/client/registry/tree.d.ts
Vendored
+10
View File
@@ -18,4 +18,14 @@ export interface ApiDefinition {
destroy: typeof routes['profile.access_tokens.destroy']
}
}
node: {
node: {
lease: typeof routes['node.node.lease']
stream: typeof routes['node.node.stream']
batch: typeof routes['node.node.batch']
heartbeat: typeof routes['node.node.heartbeat']
finish: typeof routes['node.node.finish']
release: typeof routes['node.node.release']
}
}
}