coordinator:status: the capture windows are priorities 0–2, the probe backlog from the planner's estimates
Test and deploy / test (push) Successful in 3m12s
Test and deploy / deploy (push) Successful in 2m5s

The sampling ranges in other years are capture too, at history priorities: they made a band each. The database
size walked the HDD partitions for minutes and the probe counts took 10 s on 11 M rows: dropped, and the partial
indexes' reltuples instead.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-27 14:00:08 +01:00
1 parent 680751e26a
commit c18c320927
1 file changed
+13 -10
+13 -10
View File
@@ -42,7 +42,6 @@ export interface Status {
gb: number
capGb: number
full: boolean
databaseGb: number
freeGb: number
}
bands: RangeBand[]
@@ -58,6 +57,7 @@ export interface Status {
held: HeldGroup[]
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
releases: { estimated: number; provisional: number }
/** `ready` and `waiting` are the planner's estimates (the last analyze), the others counts. */
probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
judge: { dirty: number }
nodes: { name: string; lastSeenSeconds: number | null }[]
@@ -76,20 +76,22 @@ class StatusService {
async collect(): Promise<Status> {
const tail = coordinatorConfig.tail.priority
const cap = coordinatorConfig.catalogMaxNvmeGb
// The catalog's tablespace only (CatalogSpace measures it every minute anyway): pg_database_size walks the
// chunk partitions and the probe cache on the HDD, minutes on homelab.
const { rows: space } = await db.rawQuery(
`SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes,
pg_database_size(current_database()) AS database_bytes`
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
)
const tablespaceGb = Number(space[0].bytes) / 1e9
const fs = await statfs('/').catch(() => null)
const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
// The ranges by band: the tail, the capture windows by priority, the history by its thousand.
// The ranges by band: the tail, the capture windows (priorities 0–2; the sampling ranges in other years are
// capture too, at their history priorities), the history by its thousand.
const { rows: bands } = await db.rawQuery(
`SELECT CASE WHEN priority <= ? THEN 'tail'
WHEN priority < 0 THEN 'raised'
WHEN capture THEN 'window ' || priority
WHEN capture AND priority <= 2 THEN 'window ' || priority
WHEN priority < 1000 THEN 'history 0-999'
ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
count(*) FILTER (WHERE status = 'planned') AS planned,
@@ -149,11 +151,13 @@ class StatusService {
}
files.sort((a, b) => b.mb - a.mb)
// The probe backlog from the planner's estimates of the partial indexes (counting 11 M rows took 10 s on
// homelab); the rest are small partial indexes.
const { rows: releases } = await db.rawQuery(
`SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
(SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
(SELECT count(*) FROM probe_candidates WHERE status = 'waiting' AND ready) AS ready,
(SELECT count(*) FROM probe_candidates WHERE status = 'waiting') AS waiting,
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_ready_idx'::regclass) AS ready,
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_waiting_idx'::regclass) AS waiting,
(SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
(SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
(SELECT count(*) FROM judge_dirty) AS dirty`
@@ -168,7 +172,6 @@ class StatusService {
gb: tablespaceGb,
capGb: cap,
full: cap > 0 && tablespaceGb > cap,
databaseGb: Number(space[0].database_bytes) / 1e9,
freeGb,
},
bands: bands.map((b: any) => ({
@@ -255,7 +258,7 @@ class StatusService {
lines.push(`Status at ${s.at}`)
lines.push('')
lines.push(
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; database ${s.catalog.databaseGb.toFixed(1)} GB; ${s.catalog.freeGb.toFixed(1)} GB free on /`
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; ${s.catalog.freeGb.toFixed(1)} GB free on /`
)
lines.push(
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
@@ -321,7 +324,7 @@ class StatusService {
`Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
)
lines.push(
`Probes: ${n(s.probes.ready)} ready of ${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
`Probes: ~${n(s.probes.ready)} ready of ~${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
)
lines.push(
`Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`