Seed leases respect the backfill's disk limits
No new range while the database's NVMe part or the hdd tablespace is over BACKFILL_MAX_NVME_GB / BACKFILL_MAX_HDD_GB; nodes ask again a minute later. seed:status says when leasing is paused. The NVMe part was at 47 GB of 50 with a plan for all of a.b.ath queued. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f9e9295d3e
commit
00f6187b8f
4 files changed
+33
-3
No files matched your search
@@ -90,7 +90,15 @@ class BackfillService {
|
||||
}
|
||||
|
||||
/** True when both parts of the database are under their limits, else why not. */
|
||||
private async room(maxDefaultGb: number, maxHddGb: number): Promise<true | string> {
|
||||
/**
|
||||
* Whether the database has room for more history: the default tablespace
|
||||
* (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed
|
||||
* leases (CONCEPT.md D38) check the same limits.
|
||||
*/
|
||||
async room(
|
||||
maxDefaultGb = indexerConfig.backfill.maxDefaultGb,
|
||||
maxHddGb = indexerConfig.backfill.maxHddGb
|
||||
): Promise<true | string> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
|
||||
)
|
||||
|
||||
@@ -2,6 +2,7 @@ import { createHash, randomBytes } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SeedService from '#services/SeedService'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
import { normalizeSeedFile, type SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
@@ -51,8 +52,13 @@ class SeedOrchestrator {
|
||||
return { id: Number(node.id), name: node.name }
|
||||
}
|
||||
|
||||
/** The next range for this node, oldest plan first; a quiet lease is taken over. */
|
||||
/**
|
||||
* The next range for this node, oldest plan first; a quiet lease is taken
|
||||
* over. Nothing while the database is over the backfill's disk limits: the
|
||||
* node asks again a minute later.
|
||||
*/
|
||||
async lease(nodeId: number): Promise<SeedLease | null> {
|
||||
if ((await BackfillService.room()) !== true) return null
|
||||
// Ranges that broke node after node stop being handed out.
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often')
|
||||
@@ -181,7 +187,8 @@ class SeedOrchestrator {
|
||||
const { rows: recent } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM seed_ranges WHERE done_at > now() - interval '1 hour'`
|
||||
)
|
||||
return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers) }
|
||||
const room = await BackfillService.room()
|
||||
return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers), paused: room === true ? null : room }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,5 +21,6 @@ export default class SeedStatus extends BaseCommand {
|
||||
this.logger.info(`node ${node.name}${node.disabled ? ' (disabled)' : ''}: ${node.leases} leases, ${n(node.headers)} headers, last seen ${node.last_seen_at ?? 'never'}`)
|
||||
}
|
||||
this.logger.info(`Headers in ranges finished in the last hour: ${n(status.headersDoneLastHour)}`)
|
||||
if (status.paused) this.logger.warning(`No new leases: ${status.paused} (BACKFILL_MAX_NVME_GB / BACKFILL_MAX_HDD_GB).`)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import testUtils from '@adonisjs/core/services/test_utils'
|
||||
import SeedOrchestrator from '#services/SeedOrchestrator'
|
||||
import QueueService from '#services/QueueService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { segmentFromHeader } from '#lib/ingest/segment'
|
||||
import { FileAggregator } from '#lib/aggregate/index'
|
||||
import { toFragment, toSeedFile, type SeedBatch } from '#lib/seed/wire'
|
||||
@@ -74,4 +75,17 @@ test.group('Seed API', (group) => {
|
||||
;(await client.post(`/api/v1/seed/ranges/${range.id}/release`).headers({ authorization: `Bearer ${second}` }).json({ error: 'test' })).assertStatus(200)
|
||||
assert.equal((await db.from('seed_ranges').where('id', range.id).first()).status, 'queued')
|
||||
})
|
||||
|
||||
test('no leases while the database is over the disk limit', async ({ client }) => {
|
||||
const token = await SeedOrchestrator.issueToken('full-disk')
|
||||
await db.table('seed_ranges').insert({ group_name: 'alt.binaries.test', start_id: 1, end_id: 100, next_id: 1 })
|
||||
const limit = indexerConfig.backfill.maxDefaultGb
|
||||
indexerConfig.backfill.maxDefaultGb = 0
|
||||
try {
|
||||
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(204)
|
||||
} finally {
|
||||
indexerConfig.backfill.maxDefaultGb = limit
|
||||
}
|
||||
;(await client.post('/api/v1/seed/lease').headers({ authorization: `Bearer ${token}` })).assertStatus(200)
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user