Lease lanes for the tail, a mixed lease order, coordinator:status, the edge end to end
A group now has two lease lanes (migration 6600): `history` works the ranges the bounds hand out, `tail` the tail planner's, each with its own holder, window, snapshot and takeover redo, so a thread on the tail never wades through a group's history and a held group's tail isn't stuck behind it. The edge puts `--tail-threads` (EDGE_TAIL_THREADS, 1 of 2+ by default) of its threads on the tail lane; the group routes take `?lane=`. The lease order within a lane alternates the group with the most planned articles and the longest unheld one. The takeover redo puts back only the lane's ranges the window did after the snapshot, and a group whose last range is held by a holder gone quiet can be taken over. `coordinator:status` prints the catalog's room, the ranges by band, the last hour's rates, the held groups with their snapshots and the phase-2 backlogs. `tests/e2e` runs the real edge-node binary against the node API: the corpus-fill race, a failed stream and the restart that restores the snapshot, a restart without the snapshot file, a takeover (STREAM_CHUNK_WAIT_MINUTES is a setting now). CI builds the binary before the tests. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
db96642a72
commit
680751e26a
21 files changed
+1362
-180
No files matched your search
@@ -0,0 +1,333 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { readdir, stat, statfs } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import GroupLeases from '#services/group_leases'
|
||||
import type { Lane } from '#lib/edge/index'
|
||||
|
||||
export interface RangeBand {
|
||||
band: string
|
||||
planned: number
|
||||
plannedArticles: number
|
||||
leased: number
|
||||
done: number
|
||||
doneArticles: number
|
||||
failed: number
|
||||
}
|
||||
|
||||
export interface HeldGroup {
|
||||
node: string
|
||||
group: string
|
||||
lane: Lane
|
||||
range: { start: number; end: number; next: number } | null
|
||||
heartbeatSeconds: number
|
||||
snapshotNext: number | null
|
||||
snapshotSeconds: number | null
|
||||
snapshotMb: number | null
|
||||
rangesDone: number
|
||||
headers: number
|
||||
}
|
||||
|
||||
export interface SnapshotFile {
|
||||
group: string
|
||||
lane: Lane
|
||||
mb: number
|
||||
hours: number
|
||||
}
|
||||
|
||||
export interface Status {
|
||||
at: string
|
||||
catalog: {
|
||||
tablespace: string
|
||||
gb: number
|
||||
capGb: number
|
||||
full: boolean
|
||||
databaseGb: number
|
||||
freeGb: number
|
||||
}
|
||||
bands: RangeBand[]
|
||||
lastHour: {
|
||||
indexedArticles: number
|
||||
doneRanges: number
|
||||
storedArticles: number
|
||||
storedRanges: number
|
||||
}
|
||||
corpus: { pendingRanges: number; pendingArticles: number }
|
||||
spool: { ranges: number; chunks: number }
|
||||
tail: { groups: number; ranges: number; articles: number }
|
||||
held: HeldGroup[]
|
||||
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
|
||||
releases: { estimated: number; provisional: number }
|
||||
probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
|
||||
judge: { dirty: number }
|
||||
nodes: { name: string; lastSeenSeconds: number | null }[]
|
||||
}
|
||||
|
||||
const seconds = (since: Date | string | null) =>
|
||||
since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
|
||||
|
||||
/**
|
||||
* What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
|
||||
* NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
|
||||
* with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
|
||||
* filesystem; nothing is kept between calls.
|
||||
*/
|
||||
class StatusService {
|
||||
async collect(): Promise<Status> {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
const cap = coordinatorConfig.catalogMaxNvmeGb
|
||||
const { rows: space } = await db.rawQuery(
|
||||
`SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
|
||||
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes,
|
||||
pg_database_size(current_database()) AS database_bytes`
|
||||
)
|
||||
const tablespaceGb = Number(space[0].bytes) / 1e9
|
||||
const fs = await statfs('/').catch(() => null)
|
||||
const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
|
||||
|
||||
// The ranges by band: the tail, the capture windows by priority, the history by its thousand.
|
||||
const { rows: bands } = await db.rawQuery(
|
||||
`SELECT CASE WHEN priority <= ? THEN 'tail'
|
||||
WHEN priority < 0 THEN 'raised'
|
||||
WHEN capture THEN 'window ' || priority
|
||||
WHEN priority < 1000 THEN 'history 0-999'
|
||||
ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
|
||||
count(*) FILTER (WHERE status = 'planned') AS planned,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
|
||||
count(*) FILTER (WHERE status = 'leased') AS leased,
|
||||
count(*) FILTER (WHERE status = 'done') AS done,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM ranges GROUP BY 1 ORDER BY min(priority)`,
|
||||
[tail]
|
||||
)
|
||||
const { rows: hour } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
|
||||
count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
|
||||
count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
|
||||
count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
|
||||
count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
|
||||
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
|
||||
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles
|
||||
FROM ranges`,
|
||||
[tail, tail, tail]
|
||||
)
|
||||
const h = hour[0]
|
||||
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const { rows: held } = await db.rawQuery(
|
||||
`SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
|
||||
r.start_id, r.end_id, r.next_id, l.group_id
|
||||
FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
|
||||
LEFT JOIN ranges r ON r.id = l.range_id
|
||||
WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
|
||||
ORDER BY l.lane, g.name`,
|
||||
[ttl]
|
||||
)
|
||||
const snapshotSizes = new Map<string, number>()
|
||||
const files: SnapshotFile[] = []
|
||||
const names = new Map<number, string>(
|
||||
(await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
|
||||
)
|
||||
for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
|
||||
const m = /^(\d+)(-tail)?\.gz$/.exec(file)
|
||||
if (!m) continue
|
||||
const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
|
||||
if (!s) continue
|
||||
const lane: Lane = m[2] ? 'tail' : 'history'
|
||||
snapshotSizes.set(`${m[1]}:${lane}`, s.size)
|
||||
files.push({
|
||||
group: names.get(Number(m[1])) ?? m[1],
|
||||
lane,
|
||||
mb: s.size / 1e6,
|
||||
hours: (Date.now() - s.mtimeMs) / 3600_000,
|
||||
})
|
||||
}
|
||||
files.sort((a, b) => b.mb - a.mb)
|
||||
|
||||
const { rows: releases } = await db.rawQuery(
|
||||
`SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
|
||||
(SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
|
||||
(SELECT count(*) FROM probe_candidates WHERE status = 'waiting' AND ready) AS ready,
|
||||
(SELECT count(*) FROM probe_candidates WHERE status = 'waiting') AS waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
|
||||
(SELECT count(*) FROM judge_dirty) AS dirty`
|
||||
)
|
||||
const r = releases[0]
|
||||
const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
|
||||
|
||||
return {
|
||||
at: new Date().toISOString(),
|
||||
catalog: {
|
||||
tablespace: space[0].tablespace,
|
||||
gb: tablespaceGb,
|
||||
capGb: cap,
|
||||
full: cap > 0 && tablespaceGb > cap,
|
||||
databaseGb: Number(space[0].database_bytes) / 1e9,
|
||||
freeGb,
|
||||
},
|
||||
bands: bands.map((b: any) => ({
|
||||
band: b.band,
|
||||
planned: Number(b.planned),
|
||||
plannedArticles: Number(b.planned_articles),
|
||||
leased: Number(b.leased),
|
||||
done: Number(b.done),
|
||||
doneArticles: Number(b.done_articles),
|
||||
failed: Number(b.failed),
|
||||
})),
|
||||
lastHour: {
|
||||
indexedArticles: Number(h.indexed),
|
||||
doneRanges: Number(h.done_ranges),
|
||||
storedArticles: Number(h.stored),
|
||||
storedRanges: Number(h.stored_ranges),
|
||||
},
|
||||
corpus: {
|
||||
pendingRanges: Number(h.corpus_pending),
|
||||
pendingArticles: Number(h.corpus_pending_articles),
|
||||
},
|
||||
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
|
||||
tail: {
|
||||
groups: Number(h.tail_groups),
|
||||
ranges: Number(h.tail_ranges),
|
||||
articles: Number(h.tail_articles),
|
||||
},
|
||||
held: held.map((l: any) => {
|
||||
const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
|
||||
return {
|
||||
node: l.node,
|
||||
group: l.group,
|
||||
lane: l.lane,
|
||||
range:
|
||||
l.start_id !== null
|
||||
? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
|
||||
: null,
|
||||
heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
|
||||
snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
|
||||
snapshotSeconds: seconds(l.snapshot_at),
|
||||
snapshotMb: size === undefined ? null : size / 1e6,
|
||||
rangesDone: Number(l.ranges_done),
|
||||
headers: Number(l.headers),
|
||||
}
|
||||
}),
|
||||
snapshots: {
|
||||
count: files.length,
|
||||
gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
|
||||
oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
|
||||
largest: files.slice(0, 5),
|
||||
},
|
||||
releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
|
||||
probes: {
|
||||
ready: Number(r.ready),
|
||||
waiting: Number(r.waiting),
|
||||
pendingWaiting: Number(r.pending_waiting),
|
||||
pendingNamed: Number(r.pending_named),
|
||||
},
|
||||
judge: { dirty: Number(r.dirty) },
|
||||
nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
|
||||
}
|
||||
}
|
||||
|
||||
/** The status as a text panel. */
|
||||
render(s: Status): string {
|
||||
const n = (v: number) => Math.round(v).toLocaleString('en-US')
|
||||
const b = (v: number) =>
|
||||
v >= 1e9
|
||||
? `${(v / 1e9).toFixed(2)} B`
|
||||
: v >= 1e6
|
||||
? `${(v / 1e6).toFixed(1)} M`
|
||||
: v >= 1e3
|
||||
? `${(v / 1e3).toFixed(0)} k`
|
||||
: String(v)
|
||||
const age = (sec: number | null) =>
|
||||
sec === null
|
||||
? '-'
|
||||
: sec < 90
|
||||
? `${sec} s`
|
||||
: sec < 5400
|
||||
? `${Math.round(sec / 60)} min`
|
||||
: `${(sec / 3600).toFixed(1)} h`
|
||||
const lines: string[] = []
|
||||
lines.push(`Status at ${s.at}`)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; database ${s.catalog.databaseGb.toFixed(1)} GB; ${s.catalog.freeGb.toFixed(1)} GB free on /`
|
||||
)
|
||||
lines.push(
|
||||
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
|
||||
)
|
||||
lines.push(
|
||||
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups`
|
||||
)
|
||||
lines.push('')
|
||||
const table = (header: string[], rows: string[][]) => {
|
||||
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
|
||||
const line = (r: string[]) =>
|
||||
r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(' ')
|
||||
lines.push(line(header))
|
||||
for (const r of rows) lines.push(line(r))
|
||||
}
|
||||
lines.push('Ranges by band:')
|
||||
table(
|
||||
['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
|
||||
s.bands.map((x) => [
|
||||
x.band,
|
||||
n(x.planned),
|
||||
b(x.plannedArticles),
|
||||
n(x.leased),
|
||||
n(x.done),
|
||||
b(x.doneArticles),
|
||||
n(x.failed),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(`Held groups (${s.held.length}):`)
|
||||
if (s.held.length > 0)
|
||||
table(
|
||||
[
|
||||
'group',
|
||||
'lane',
|
||||
'node',
|
||||
'range',
|
||||
'at',
|
||||
'heartbeat',
|
||||
'snapshot',
|
||||
'size',
|
||||
'done',
|
||||
'headers',
|
||||
],
|
||||
s.held.map((x) => [
|
||||
x.group,
|
||||
x.lane,
|
||||
x.node,
|
||||
x.range ? `${x.range.start}-${x.range.end}` : '-',
|
||||
x.range ? String(x.range.next) : '-',
|
||||
age(x.heartbeatSeconds),
|
||||
x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
|
||||
x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
|
||||
n(x.rangesDone),
|
||||
b(x.headers),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
|
||||
)
|
||||
lines.push(
|
||||
`Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
|
||||
)
|
||||
lines.push(
|
||||
`Probes: ${n(s.probes.ready)} ready of ${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
|
||||
)
|
||||
lines.push(
|
||||
`Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
|
||||
)
|
||||
return lines.join('\n')
|
||||
}
|
||||
}
|
||||
|
||||
export default new StatusService()
|
||||
Reference in new issue
Block a user