Rebuild: fresh AdonisJS 7 API app on Postgres
The rebuild branch starts over (REBUILD.md): the API starter kit, with Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a shared one in development) and the #lib alias. Phase 1's code is imported from the current app module by module. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
699395c51f
commit
96f0ade10e
264 files changed
+777
-24132
No files matched your search
@@ -32,6 +32,10 @@ export const http = defineConfig({
|
||||
*/
|
||||
useAsyncLocalStorage: false,
|
||||
|
||||
router: {
|
||||
matcher: 'tree',
|
||||
},
|
||||
|
||||
/**
|
||||
* Redirect configuration controls the behavior of
|
||||
* response.redirect().back() and query string forwarding.
|
||||
|
||||
@@ -32,11 +32,6 @@ const bodyParserConfig = defineConfig({
|
||||
*/
|
||||
convertEmptyStringsToNull: true,
|
||||
|
||||
/**
|
||||
* Seeding nodes post batches of a few MB (CONCEPT.md D38); the limit applies after gzip.
|
||||
*/
|
||||
limit: '64mb',
|
||||
|
||||
/**
|
||||
* Content types handled by the JSON parser.
|
||||
*/
|
||||
|
||||
+14
-115
@@ -1,137 +1,36 @@
|
||||
import env from '#start/env'
|
||||
import app from '@adonisjs/core/services/app'
|
||||
import { defineConfig } from '@adonisjs/lucid'
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* The rebuild's own database (REBUILD.md, decision b), on the same server
|
||||
* as the current indexer's.
|
||||
*/
|
||||
const dbConfig = defineConfig({
|
||||
/**
|
||||
* Default connection used for all queries.
|
||||
*/
|
||||
connection: env.get('DB_CONNECTION', 'sqlite'),
|
||||
|
||||
connection: 'pg',
|
||||
connections: {
|
||||
/**
|
||||
* SQLite connection (default).
|
||||
*/
|
||||
sqlite: {
|
||||
client: 'better-sqlite3',
|
||||
|
||||
connection: {
|
||||
filename: app.tmpPath('db.sqlite3'),
|
||||
},
|
||||
|
||||
/**
|
||||
* Required by Knex for SQLite defaults.
|
||||
*/
|
||||
useNullAsDefault: true,
|
||||
|
||||
migrations: {
|
||||
/**
|
||||
* Sort migration files naturally by filename.
|
||||
*/
|
||||
naturalSort: true,
|
||||
|
||||
/**
|
||||
* Paths containing migration files.
|
||||
*/
|
||||
paths: ['database/migrations'],
|
||||
},
|
||||
|
||||
schemaGeneration: {
|
||||
/**
|
||||
* Enable schema generation from Lucid models.
|
||||
*/
|
||||
enabled: true,
|
||||
|
||||
/**
|
||||
* Custom schema rules file paths.
|
||||
*/
|
||||
rulesPaths: ['./database/schema_rules.js'],
|
||||
},
|
||||
},
|
||||
|
||||
/**
|
||||
* PostgreSQL connection.
|
||||
* Install package to switch: npm install pg
|
||||
*/
|
||||
pg: {
|
||||
client: 'pg',
|
||||
connection: {
|
||||
host: env.get('DB_HOST'),
|
||||
port: env.get('DB_PORT'),
|
||||
user: env.get('DB_USER'),
|
||||
password: env.get('DB_PASSWORD')?.release(),
|
||||
password: env.get('DB_PASSWORD'),
|
||||
database: env.get('DB_DATABASE'),
|
||||
// For idempotent bulk writers (worker:fetch): don't wait for the WAL
|
||||
// flush on every statement. A crash loses the last moments of writes,
|
||||
// which the retried job writes again.
|
||||
...(env.get('DB_SYNCHRONOUS_COMMIT') === 'off' ? { options: '-c synchronous_commit=off' } : {}),
|
||||
},
|
||||
// One connection per concurrent job; knex defaults to 10.
|
||||
// Local development may share a database, in its own schema (DB_SEARCH_PATH=rebuild).
|
||||
...(env.get('DB_SEARCH_PATH') ? { searchPath: env.get('DB_SEARCH_PATH')!.split(',') } : {}),
|
||||
pool: { min: 0, max: env.get('DB_POOL_MAX', 10) },
|
||||
migrations: {
|
||||
naturalSort: true,
|
||||
paths: ['database/migrations'],
|
||||
},
|
||||
debug: app.inDev,
|
||||
schemaGeneration: {
|
||||
enabled: true,
|
||||
rulesPaths: ['./database/schema_rules.js'],
|
||||
},
|
||||
debug: app.inDev && env.get('DB_DEBUG', false),
|
||||
},
|
||||
|
||||
/**
|
||||
* MySQL / MariaDB connection.
|
||||
* Install package to switch: npm install mysql2
|
||||
*/
|
||||
// mysql: {
|
||||
// client: 'mysql2',
|
||||
// connection: {
|
||||
// host: env.get('DB_HOST'),
|
||||
// port: env.get('DB_PORT'),
|
||||
// user: env.get('DB_USER'),
|
||||
// password: env.get('DB_PASSWORD'),
|
||||
// database: env.get('DB_DATABASE'),
|
||||
// },
|
||||
// migrations: {
|
||||
// naturalSort: true,
|
||||
// paths: ['database/migrations'],
|
||||
// },
|
||||
// debug: app.inDev,
|
||||
// },
|
||||
|
||||
/**
|
||||
* Microsoft SQL Server connection.
|
||||
* Install package to switch: npm install tedious
|
||||
*/
|
||||
// mssql: {
|
||||
// client: 'mssql',
|
||||
// connection: {
|
||||
// server: env.get('DB_HOST'),
|
||||
// port: env.get('DB_PORT'),
|
||||
// user: env.get('DB_USER'),
|
||||
// password: env.get('DB_PASSWORD'),
|
||||
// database: env.get('DB_DATABASE'),
|
||||
// },
|
||||
// migrations: {
|
||||
// naturalSort: true,
|
||||
// paths: ['database/migrations'],
|
||||
// },
|
||||
// debug: app.inDev,
|
||||
// },
|
||||
|
||||
/**
|
||||
* libSQL (Turso) connection.
|
||||
* Install package to switch: npm install @libsql/client
|
||||
*/
|
||||
// libsql: {
|
||||
// client: 'libsql',
|
||||
// connection: {
|
||||
// url: env.get('LIBSQL_URL'),
|
||||
// authToken: env.get('LIBSQL_AUTH_TOKEN'),
|
||||
// },
|
||||
// useNullAsDefault: true,
|
||||
// migrations: {
|
||||
// naturalSort: true,
|
||||
// paths: ['database/migrations'],
|
||||
// },
|
||||
// debug: app.inDev,
|
||||
// },
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* Retention for transient pipeline state in Redis. See CONCEPT.md (D11).
|
||||
*/
|
||||
export default {
|
||||
/** How often `gc:scheduler` sweeps. */
|
||||
intervalMs: env.get('GC_INTERVAL_MS', 3 * 60 * 1000),
|
||||
|
||||
/** How often gc:scheduler recomputes the completeness of releases that changed (LOG.md 2026-10-06 (6)). */
|
||||
releaseCompleteIntervalMs: env.get('RELEASE_COMPLETE_INTERVAL_MS', 30 * 60 * 1000),
|
||||
|
||||
/** How often gc:scheduler merges uploads split per poster (D35); it reads every release. */
|
||||
mergeIntervalMs: env.get('GC_MERGE_INTERVAL_MS', 60 * 60 * 1000),
|
||||
|
||||
/** Time per sweep for copying old file keys into file_keys until it's complete (LOG.md 2026-10-06 (7)). */
|
||||
fileKeysFillMs: env.get('FILE_KEYS_FILL_SECONDS', 90) * 1000,
|
||||
|
||||
jobs: {
|
||||
/** Completed jobs are only kept for inspection; keep them briefly. */
|
||||
completedAgeSeconds: env.get('GC_COMPLETED_AGE_SECONDS', 60 * 60),
|
||||
completedMaxCount: 1000,
|
||||
/** Failed jobs are kept longer so they can be inspected and retried. */
|
||||
failedAgeSeconds: env.get('GC_FAILED_AGE_SECONDS', 7 * 24 * 60 * 60),
|
||||
},
|
||||
|
||||
/**
|
||||
* Upper bound on waiting body jobs. Obfuscated traffic can exceed what the
|
||||
* line downloads for hours; beyond this, the oldest waiting jobs are
|
||||
* dropped so Redis stays bounded (~1 KB per job).
|
||||
*/
|
||||
bodyQueueMax: env.get('GC_BODY_QUEUE_MAX', 500_000),
|
||||
|
||||
binaries: {
|
||||
/**
|
||||
* A binary that gets no new segments for this long is deleted: incomplete
|
||||
* ones are given up on, completed ones have been copied into `files`.
|
||||
*/
|
||||
staleAfterSeconds: env.get('GC_BINARY_STALE_SECONDS', 48 * 60 * 60),
|
||||
/**
|
||||
* Header-derived binaries that still have one article after this long:
|
||||
* uploads that give every article its own random name (and poster) never
|
||||
* complete from headers. A real file gets its next articles in minutes.
|
||||
*/
|
||||
singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60),
|
||||
/**
|
||||
* Completed header-derived binaries without a fingerprint (size, MD5 of
|
||||
* the first 16 KiB): their file exists and no par2 set can name them
|
||||
* later, so they go sooner than staleAfterSeconds.
|
||||
*/
|
||||
unfingerprintedAfterSeconds: env.get('GC_UNFINGERPRINTED_SECONDS', 6 * 60 * 60),
|
||||
/**
|
||||
* A finished binary whose file waits for content inspection is kept this long at
|
||||
* most (it needs the fingerprint inspection stores for par2 naming, D16).
|
||||
*/
|
||||
awaitInspectionSeconds: env.get('GC_AWAIT_INSPECTION_SECONDS', 7 * 24 * 60 * 60),
|
||||
},
|
||||
|
||||
/**
|
||||
* Staged lone segments (CONCEPT.md D35) are dropped after this long. A
|
||||
* real file's next segment comes in the same or the next fetch range,
|
||||
* seconds to minutes later; at backfill speed two hours of waiting
|
||||
* segments were 12M rows, 10 GB.
|
||||
*/
|
||||
stagingTtlSeconds: env.get('STAGING_TTL_SECONDS', 30 * 60),
|
||||
|
||||
/** Releases without files or binaries (their binaries were collected) are deleted after this long. */
|
||||
emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60),
|
||||
|
||||
/**
|
||||
* Bulk deletes run in batches of this size, each kind (copied parts,
|
||||
* single-article and unfingerprinted binaries, empty releases) for at most
|
||||
* this long per sweep, so one backlog can't starve the others.
|
||||
*/
|
||||
deleteBatch: 20_000,
|
||||
deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 60 * 1000),
|
||||
/** Bloated indexes are rebuilt at most this often (IndexMaintenance). */
|
||||
reindexIntervalMs: env.get('GC_REINDEX_INTERVAL_MS', 7 * 24 * 60 * 60 * 1000),
|
||||
|
||||
/**
|
||||
* Deferred files (CONCEPT.md D28) are deleted this long after they were
|
||||
* first seen, processed or not. The articles stay on the server.
|
||||
*/
|
||||
deferredRetentionDays: env.get('GC_DEFERRED_RETENTION_DAYS', 30),
|
||||
|
||||
/** Redis hashes from the old header worker (before D12); they only need to expire. */
|
||||
legacyPendingFiles: {
|
||||
prefix: 'file:',
|
||||
ttlSeconds: 48 * 60 * 60,
|
||||
},
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
/** How often `index:scheduler` checks the groups for new articles. */
|
||||
intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
|
||||
/** Articles per fetch job (one XOVER call). */
|
||||
batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
|
||||
/** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
|
||||
maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
|
||||
/**
|
||||
* Where a group that was never indexed starts: this many articles before
|
||||
* the newest one. The server's low-water mark can be years back.
|
||||
*/
|
||||
initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),
|
||||
|
||||
/**
|
||||
* Obfuscated articles whose message-id matches one of these are not sent
|
||||
* to the body queue: their uploader spreads every file over many groups,
|
||||
* so identifying them would take far more bandwidth than there is
|
||||
* (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
|
||||
*/
|
||||
skipBodyMessageIds: [
|
||||
// hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
|
||||
/^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
|
||||
],
|
||||
|
||||
/**
|
||||
* Obfuscated articles whose message-id matches one of these don't go to
|
||||
* the body queue. Instead they're recorded per file in `deferred_files`
|
||||
* (their subject keeps the file name, only the part counter is missing).
|
||||
* `worker:deferred` later fetches one body per file (CONCEPT.md D28).
|
||||
*/
|
||||
deferBodyMessageIds: [
|
||||
// ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
|
||||
/^<?[0-9a-f]{32}@ngPost>?$/,
|
||||
],
|
||||
/**
|
||||
* Spotnet header groups (CONCEPT.md D29): their headers are spots, not
|
||||
* binaries. Add one with `group:add free.pt --backfill 5000`.
|
||||
*/
|
||||
spotGroups: ['free.pt'],
|
||||
/**
|
||||
* Spots posted within this many days are imported (their NZB read and its
|
||||
* files added). Older spots are listed: title, size and date, with the NZB
|
||||
* read on first download.
|
||||
*/
|
||||
spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),
|
||||
|
||||
/**
|
||||
* Groups whose lone header segments wait in `staged_segments` until their
|
||||
* file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
|
||||
*/
|
||||
stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
|
||||
|
||||
/**
|
||||
* Groups whose staged segments also group by poster, file counter `[i/M]`
|
||||
* and part count (CONCEPT.md D36), for uploads with a random subject per
|
||||
* article. Needs staging. Comma-separated; `*` for all.
|
||||
*/
|
||||
counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
|
||||
|
||||
/**
|
||||
* Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
|
||||
* header-only fetch ranges walking back through a group's history.
|
||||
*/
|
||||
seed: {
|
||||
/** NNTP connections the seed API fetches with for relay nodes (CONCEPT.md D38); 0: relay off. */
|
||||
nntpConnections: env.get('SEED_NNTP_CONNECTIONS', 0),
|
||||
/**
|
||||
* Silence that counts as a dead connection during a relay XOVER (an
|
||||
* inactivity timeout: slow ranges keep sending). Chunks that "never
|
||||
* answered" in 10 minutes came in 0.3 s on a fresh connection.
|
||||
*/
|
||||
nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 90 * 1000),
|
||||
/**
|
||||
* Ask the news server for gzip-compressed overviews (SEED_NNTP_COMPRESS=false: plain).
|
||||
* On one connection plain had a shorter tail for 2026 articles, but with 20 relay
|
||||
* connections both gave ~1.5 chunks/s, plain at many times the relay's CPU and
|
||||
* more traffic (LOG.md 2026-10-06 (9)).
|
||||
*/
|
||||
nntpCompress: env.get('SEED_NNTP_COMPRESS', true),
|
||||
},
|
||||
|
||||
backfill: {
|
||||
/** Backfill ranges waiting or running at a time, over all groups. */
|
||||
maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
|
||||
/** BullMQ priority (higher runs later; live ranges have none, so they go first). */
|
||||
priority: 10,
|
||||
/**
|
||||
* No new backfill ranges while the database's NVMe part (default
|
||||
* tablespace) or its HDD part (tablespace `hdd`, the files table) is
|
||||
* larger than this. Checked every scheduler tick.
|
||||
*/
|
||||
maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
|
||||
maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
|
||||
},
|
||||
|
||||
/**
|
||||
* Dropping garbage uploads early (CONCEPT.md D42). A signature (posting tool, first
|
||||
* group) is dropped once at least minProbes of its probed random-named releases are
|
||||
* known and at most maxNamedRate of them got a real name or password; its
|
||||
* random-named files are then not stored, except explorePercent of its releases.
|
||||
*/
|
||||
garbage: {
|
||||
/** Off: nothing is filtered or deleted (the learning still runs). */
|
||||
enabled: env.get('GARBAGE_FILTER', true),
|
||||
explorePercent: env.get('GARBAGE_EXPLORE_PERCENT', 2),
|
||||
minProbes: 100,
|
||||
maxNamedRate: 0.005,
|
||||
/** Probed releases learned from: 1 in this many (by release id), for the last windowDays. */
|
||||
probeSample: 4,
|
||||
windowDays: 30,
|
||||
/** How often gc:scheduler learns the signatures again. */
|
||||
learnIntervalMs: env.get('GARBAGE_LEARN_INTERVAL_MS', 3 * 60 * 60 * 1000),
|
||||
},
|
||||
|
||||
/** worker:deferred leaves a file alone until it has had no new articles for this long. */
|
||||
deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
|
||||
/** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
|
||||
deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
host: env.get('NNTP_HOST'),
|
||||
port: env.get('NNTP_PORT', 119),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', false),
|
||||
/** Connections per process. Mind the provider's limit across all processes. */
|
||||
poolSize: env.get('NNTP_POOL_SIZE', 10),
|
||||
/** A command without an answer this long counts as a dead connection (XOVER of 20k headers takes ~0.2 s). */
|
||||
commandTimeoutMs: env.get('NNTP_COMMAND_TIMEOUT_MS', 120_000),
|
||||
/** TLS certificate hostname when NNTP_HOST is a tunnel (seeding nodes; lib/nntp/xover_client). */
|
||||
tlsServername: env.get('NNTP_TLS_SERVERNAME'),
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
connection: {
|
||||
host: env.get('REDIS_HOST', '127.0.0.1'),
|
||||
port: env.get('REDIS_PORT', 6379),
|
||||
},
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
connection: 'main',
|
||||
connections: {
|
||||
main: {
|
||||
host: env.get('REDIS_HOST', '127.0.0.1'),
|
||||
port: env.get('REDIS_PORT'),
|
||||
password: env.get('REDIS_PASSWORD')?.release(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import { defineConfig } from '@adonisjs/static'
|
||||
|
||||
/**
|
||||
* Configuration options to tweak the static files middleware.
|
||||
* The complete set of options are documented on the
|
||||
* official documentation website.
|
||||
*
|
||||
* https://docs.adonisjs.com/guides/static-assets
|
||||
*/
|
||||
const staticServerConfig = defineConfig({
|
||||
enabled: true,
|
||||
etag: true,
|
||||
lastModified: true,
|
||||
dotFiles: 'ignore',
|
||||
})
|
||||
|
||||
export default staticServerConfig
|
||||
Reference in new issue
Block a user