Content inspection and archive walks on seeding nodes (relay)
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51975fe603
commit
2dbca9b152
22 files changed
+1198
-540
No files matched your search
@@ -11,6 +11,38 @@
|
||||
|
||||
import { Env } from '@adonisjs/core/env'
|
||||
|
||||
/**
|
||||
* A seeding node (`node ace seed:node`, CONCEPT.md D38) has no database,
|
||||
* Redis or web server, and in relay mode no NNTP account, but the app's
|
||||
* configuration still reads those settings: they get placeholders here, so
|
||||
* a node needs only SEED_URL and SEED_TOKEN. Values that are set win (e.g.
|
||||
* the NNTP account of a node with SEED_FETCH=direct).
|
||||
*/
|
||||
if (process.env.ACE_COMMAND === 'seed:node') {
|
||||
const placeholders: Record<string, string> = {
|
||||
NODE_ENV: 'production',
|
||||
PORT: '3333',
|
||||
HOST: '0.0.0.0',
|
||||
LOG_LEVEL: 'info',
|
||||
APP_KEY: 'unused-on-seed-nodes-unused-on-seed-nodes',
|
||||
APP_URL: 'http://localhost:3333',
|
||||
SESSION_DRIVER: 'memory',
|
||||
DB_CONNECTION: 'pg',
|
||||
DB_HOST: 'localhost',
|
||||
DB_PORT: '5432',
|
||||
DB_USER: 'unused',
|
||||
DB_DATABASE: 'unused',
|
||||
REDIS_HOST: 'localhost',
|
||||
REDIS_PORT: '6379',
|
||||
NNTP_HOST: 'localhost',
|
||||
NNTP_PORT: '563',
|
||||
NNTP_USER: 'unused',
|
||||
NNTP_PASSWORD: 'unused',
|
||||
NNTP_SECURE: 'true',
|
||||
}
|
||||
for (const [name, value] of Object.entries(placeholders)) process.env[name] ??= value
|
||||
}
|
||||
|
||||
export default await Env.create(new URL('../', import.meta.url), {
|
||||
// Node
|
||||
NODE_ENV: Env.schema.enum(['development', 'production', 'test'] as const),
|
||||
|
||||
@@ -59,6 +59,12 @@ router
|
||||
router.post('ranges/:id/done', [SeedController, 'done']).where('id', router.matchers.number())
|
||||
router.post('ranges/:id/release', [SeedController, 'release']).where('id', router.matchers.number())
|
||||
router.get('status', [SeedController, 'status'])
|
||||
// Content inspection and archive walks on nodes (CONCEPT.md D41).
|
||||
router.post('inspect/lease', [SeedController, 'inspectLease'])
|
||||
router.post('inspect/done', [SeedController, 'inspectDone'])
|
||||
router.post('inspect/failed', [SeedController, 'inspectFailed'])
|
||||
router.post('inspect/release', [SeedController, 'inspectRelease'])
|
||||
router.get('articles', [SeedController, 'article'])
|
||||
})
|
||||
.prefix('/api/v1/seed')
|
||||
.use(middleware.seedNode())
|
||||
Reference in new issue
Block a user