From 00f6187b8fbf667767f28d536c119074e3d1f314 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Mon, 5 Oct 2026 13:30:46 +0200 Subject: [PATCH] Seed leases respect the backfill's disk limits No new range while the database's NVMe part or the hdd tablespace is over BACKFILL_MAX_NVME_GB / BACKFILL_MAX_HDD_GB; nodes ask again a minute later. seed:status says when leasing is paused. The NVMe part was at 47 GB of 50 with a plan for all of a.b.ath queued. Co-Authored-By: Claude Opus 5.5 --- app/services/BackfillService.ts | 10 +++++++++- app/services/SeedOrchestrator.ts | 11 +++++++++-- commands/SeedStatus.ts | 1 + tests/functional/seed_api.spec.ts | 14 ++++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/app/services/BackfillService.ts b/app/services/BackfillService.ts index 82e4b77..600a3de 100644 --- a/app/services/BackfillService.ts +++ b/app/services/BackfillService.ts @@ -90,7 +90,15 @@ class BackfillService { } /** True when both parts of the database are under their limits, else why not. */ - private async room(maxDefaultGb: number, maxHddGb: number): Promise { + /** + * Whether the database has room for more history: the default tablespace + * (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed + * leases (CONCEPT.md D38) check the same limits. + */ + async room( + maxDefaultGb = indexerConfig.backfill.maxDefaultGb, + maxHddGb = indexerConfig.backfill.maxHddGb + ): Promise { const { rows } = await db.rawQuery( `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')` ) diff --git a/app/services/SeedOrchestrator.ts b/app/services/SeedOrchestrator.ts index b0da532..4b90cf8 100644 --- a/app/services/SeedOrchestrator.ts +++ b/app/services/SeedOrchestrator.ts @@ -2,6 +2,7 @@ import { createHash, randomBytes } from 'node:crypto' import db from '@adonisjs/lucid/services/db' import NntpService from '#services/NntpService' import SeedService from '#services/SeedService' +import BackfillService from '#services/BackfillService' import { articleAtDate } from '#lib/nntp/date_search' import { normalizeSeedFile, type SeedBatch } from '#lib/seed/wire' @@ -51,8 +52,13 @@ class SeedOrchestrator { return { id: Number(node.id), name: node.name } } - /** The next range for this node, oldest plan first; a quiet lease is taken over. */ + /** + * The next range for this node, oldest plan first; a quiet lease is taken + * over. Nothing while the database is over the backfill's disk limits: the + * node asks again a minute later. + */ async lease(nodeId: number): Promise { + if ((await BackfillService.room()) !== true) return null // Ranges that broke node after node stop being handed out. await db.rawQuery( `UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often') @@ -181,7 +187,8 @@ class SeedOrchestrator { const { rows: recent } = await db.rawQuery( `SELECT coalesce(sum(headers), 0)::bigint AS headers FROM seed_ranges WHERE done_at > now() - interval '1 hour'` ) - return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers) } + const room = await BackfillService.room() + return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers), paused: room === true ? null : room } } } diff --git a/commands/SeedStatus.ts b/commands/SeedStatus.ts index 599205f..a182579 100644 --- a/commands/SeedStatus.ts +++ b/commands/SeedStatus.ts @@ -21,5 +21,6 @@ export default class SeedStatus extends BaseCommand { this.logger.info(`node ${node.name}${node.disabled ? ' (disabled)' : ''}: ${node.leases} leases, ${n(node.headers)} headers, last seen ${node.last_seen_at ?? 'never'}`) } this.logger.info(`Headers in ranges finished in the last hour: ${n(status.headersDoneLastHour)}`) + if (status.paused) this.logger.warning(`No new leases: ${status.paused} (BACKFILL_MAX_NVME_GB / BACKFILL_MAX_HDD_GB).`) } } diff --git a/tests/functional/seed_api.spec.ts b/tests/functional/seed_api.spec.ts index 8c585be..60c1a63 100644 --- a/tests/functional/seed_api.spec.ts +++ b/tests/functional/seed_api.spec.ts @@ -4,6 +4,7 @@ import db from '@adonisjs/lucid/services/db' import testUtils from '@adonisjs/core/services/test_utils' import SeedOrchestrator from '#services/SeedOrchestrator' import QueueService from '#services/QueueService' +import indexerConfig from '#config/indexer' import { segmentFromHeader } from '#lib/ingest/segment' import { FileAggregator } from '#lib/aggregate/index' import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire' @@ -74,4 +75,17 @@ test.group('Seed API', (group) => { ;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers({ authorization: `Bearer ${second}` }).json({ error: 'test' })).assertStatus(200) assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'queued') }) + + test('no leases while the database is over the disk limit', async ({ client }) => { + const token = await SeedOrchestrator.issueToken('full-disk') + await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 }) + const limit = indexerConfig.backfill.maxDefaultGb + indexerConfig.backfill.maxDefaultGb = 0 + try { + ;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204) + } finally { + indexerConfig.backfill.maxDefaultGb = limit + } + ;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(200) + }) })