An "ancient" range a group: articles 1 … low watermark − 1, never filled
Test and deploy / deploy (push) Successful in 3m49s
Test and deploy / test (push) Successful in 3m19s

Every group is tiled from article 1 now: one range with status 'ancient'
covers what had expired on the server before the group was planned.
Nothing selects it (leases, prefetcher, corpus fill, tail merger all take
'planned'), the status panel reports it apart, the search's history stats
leave it out. Migration 6700 adds it for the groups planned before;
Planner.historyRows (now a pure function, unit-tested) for new groups.
The status bands are ordered by an explicit key: the ancient row at
priority 0 tied "history 0-999" with "window 0".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-28 09:12:05 +01:00
1 parent c18c320927
commit 68b480c97e
7 files changed
+119 -20

No files matched your search

+11 -4
View File
@@ -54,6 +54,8 @@ export interface Status {
corpus: { pendingRanges: number; pendingArticles: number }
spool: { ranges: number; chunks: number }
tail: { groups: number; ranges: number; articles: number }
/** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
ancient: { groups: number; articles: number }
held: HeldGroup[]
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
releases: { estimated: number; provisional: number }
@@ -100,8 +102,10 @@ class StatusService {
count(*) FILTER (WHERE status = 'done') AS done,
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
count(*) FILTER (WHERE status = 'failed') AS failed
FROM ranges GROUP BY 1 ORDER BY min(priority)`,
[tail]
FROM ranges GROUP BY 1
ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
[tail, tail]
)
const { rows: hour } = await db.rawQuery(
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
@@ -114,7 +118,9 @@ class StatusService {
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
FROM ranges`,
[tail, tail, tail]
)
@@ -194,6 +200,7 @@ class StatusService {
pendingArticles: Number(h.corpus_pending_articles),
},
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
tail: {
groups: Number(h.tail_groups),
ranges: Number(h.tail_ranges),
@@ -264,7 +271,7 @@ class StatusService {
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
)
lines.push(
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups`
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
)
lines.push('')
const table = (header: string[], rows: string[][]) => {