An "ancient" range a group: articles 1 … low watermark − 1, never filled
Every group is tiled from article 1 now: one range with status 'ancient' covers what had expired on the server before the group was planned. Nothing selects it (leases, prefetcher, corpus fill, tail merger all take 'planned'), the status panel reports it apart, the search's history stats leave it out. Migration 6700 adds it for the groups planned before; Planner.historyRows (now a pure function, unit-tested) for new groups. The status bands are ordered by an explicit key: the ancient row at priority 0 tied "history 0-999" with "window 0". Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
c18c320927
commit
68b480c97e
7 files changed
+119
-20
No files matched your search
+11
-4
@@ -54,6 +54,8 @@ export interface Status {
|
||||
corpus: { pendingRanges: number; pendingArticles: number }
|
||||
spool: { ranges: number; chunks: number }
|
||||
tail: { groups: number; ranges: number; articles: number }
|
||||
/** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
|
||||
ancient: { groups: number; articles: number }
|
||||
held: HeldGroup[]
|
||||
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
|
||||
releases: { estimated: number; provisional: number }
|
||||
@@ -100,8 +102,10 @@ class StatusService {
|
||||
count(*) FILTER (WHERE status = 'done') AS done,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM ranges GROUP BY 1 ORDER BY min(priority)`,
|
||||
[tail]
|
||||
FROM ranges GROUP BY 1
|
||||
ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
|
||||
WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
|
||||
[tail, tail]
|
||||
)
|
||||
const { rows: hour } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
|
||||
@@ -114,7 +118,9 @@ class StatusService {
|
||||
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
|
||||
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
|
||||
FROM ranges`,
|
||||
[tail, tail, tail]
|
||||
)
|
||||
@@ -194,6 +200,7 @@ class StatusService {
|
||||
pendingArticles: Number(h.corpus_pending_articles),
|
||||
},
|
||||
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
|
||||
ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
|
||||
tail: {
|
||||
groups: Number(h.tail_groups),
|
||||
ranges: Number(h.tail_ranges),
|
||||
@@ -264,7 +271,7 @@ class StatusService {
|
||||
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
|
||||
)
|
||||
lines.push(
|
||||
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups`
|
||||
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
|
||||
)
|
||||
lines.push('')
|
||||
const table = (header: string[], rows: string[][]) => {
|
||||
|
||||
Reference in new issue
Block a user