diff --git a/app/services/disk_space.ts b/app/services/disk_space.ts new file mode 100644 index 0000000..30a5da6 --- /dev/null +++ b/app/services/disk_space.ts @@ -0,0 +1,40 @@ +import { statfs } from 'node:fs/promises' +import logger from '@adonisjs/core/services/logger' + +/** How often free space is measured. */ +const CHECK_MS = 60_000 +/** Resumed only this much above the floor: not paused and resumed at every flush. */ +const MARGIN_GB = 1 + +/** + * Free space on the filesystem holding `path`, against a floor: low below it, again fine MARGIN_GB above it. In a + * container `/` is an overlay on the host's Docker root, which on homelab is the NVMe the catalog and the WAL are on. + * The prober filled it ~0.8–1.1 GB an hour at ~200 articles/s, and nothing stopped it at the 10 GB floor (LOG.md + * 2026-10-18 (77)). Off when the floor is 0. + */ +export class DiskSpace { + private checkedAt = 0 + private low = false + + constructor( + private what: string, + private path: string, + private floorGb: number, + private free: (path: string) => Promise = async (p) => { + const s = await statfs(p) + return s.bavail * s.bsize + }, + private checkMs = CHECK_MS + ) {} + + async short(): Promise { + if (!this.floorGb) return false + if (Date.now() - this.checkedAt < this.checkMs) return this.low + this.checkedAt = Date.now() + const gb = (await this.free(this.path)) / 1e9 + const was = this.low + this.low = gb < this.floorGb + (was ? MARGIN_GB : 0) + if (this.low !== was) logger.info(`${gb.toFixed(1)} GB free (floor ${this.floorGb} GB): ${this.what} ${this.low ? 'paused' : 'resumed'}.`) + return this.low + } +} diff --git a/app/services/prober.ts b/app/services/prober.ts index 650c861..e2efaa4 100644 --- a/app/services/prober.ts +++ b/app/services/prober.ts @@ -7,6 +7,7 @@ import { XoverClient } from '#lib/nntp/index' import { YencError } from '#lib/yenc/index' import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index' import { NntpShare } from '#services/nntp_shares' +import { DiskSpace } from '#services/disk_space' interface Candidate { id: number @@ -52,6 +53,8 @@ export class Prober { private running = 0 /** The last lane to look found nothing to claim: the prober wants its minimum only. */ private dry = false + /** What it stores (inspections, par2 file lists, the WAL) fills the NVMe: no claims below PROBE_MIN_FREE_GB. */ + private space = new DiskSpace('probing', '/', coordinatorConfig.probe.minFreeGb) private share = new NntpShare('prober', coordinatorConfig.nntpShares.proberPriority, coordinatorConfig.probe.minConnections, coordinatorConfig.probe.connections) /** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */ @@ -201,6 +204,8 @@ export class Prober { } private async claimTiers() { + // The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again. + if (await this.space.short()) return for (const { kind, unreadable } of TIERS) { const want = this.batch - this.queue.length if (want <= 0) return diff --git a/config/coordinator.ts b/config/coordinator.ts index 57ca3a8..3b729c8 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -90,6 +90,8 @@ export default { connections: env.get('PROBE_CONNECTIONS', 6), /** Of them, always (its share's minimum; nntpShares). */ minConnections: env.get('PROBE_MIN_CONNECTIONS', 2), + /** No claims while the filesystem under `/` (the NVMe, in the container) has less free (app/services/disk_space.ts). 0: off. */ + minFreeGb: env.get('PROBE_MIN_FREE_GB', 0), /** Candidates claimed at a time. */ batch: 400, /** BODY commands in flight per connection (pipelined; XoverClient.bodies). */ diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 9a5dd3c..e6e5238 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -130,6 +130,9 @@ services: environment: <<: *app-env PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-8} + # No claims under this much free on the NVMe (its overlay root): 2 GB above the 10 GB floor, for what the + # tail, the merger and the judge write meanwhile. + PROBE_MIN_FREE_GB: ${PROBE_MIN_FREE_GB:-12} # The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP. merger: diff --git a/start/env.ts b/start/env.ts index 75f3579..3c49297 100644 --- a/start/env.ts +++ b/start/env.ts @@ -78,6 +78,7 @@ export default await Env.create(new URL('../', import.meta.url), { PREFETCH_PER_RANGE: Env.schema.number.optional(), PROBE_CONNECTIONS: Env.schema.number.optional(), PROBE_MIN_CONNECTIONS: Env.schema.number.optional(), + PROBE_MIN_FREE_GB: Env.schema.number.optional(), HTTP_ROUTES: Env.schema.enum.optional(['all', 'node', 'public'] as const), PREFETCH_PER_GROUP: Env.schema.number.optional(), }) diff --git a/tests/unit/disk_space.spec.ts b/tests/unit/disk_space.spec.ts new file mode 100644 index 0000000..bb8c468 --- /dev/null +++ b/tests/unit/disk_space.spec.ts @@ -0,0 +1,22 @@ +import { test } from '@japa/runner' +import { DiskSpace } from '#services/disk_space' + +test.group('DiskSpace', () => { + test('short below the floor, fine again a margin above it; off without a floor', async ({ assert }) => { + let gb = 15 + const space = new DiskSpace('probing', '/', 12, async () => gb * 1e9, 0) + assert.isFalse(await space.short()) + gb = 11.5 + assert.isTrue(await space.short()) + // Back over the floor, not yet over the margin: still short. + gb = 12.5 + assert.isTrue(await space.short()) + gb = 13.1 + assert.isFalse(await space.short()) + assert.isFalse(await new DiskSpace('probing', '/', 0, async () => 0, 0).short()) + }) + + test('measures a real filesystem', async ({ assert }) => { + assert.isFalse(await new DiskSpace('probing', '/', 0.001).short()) + }) +})