Rebuild: the live tail

coordinator:run plans the articles after each group's last range every
TAIL_EVERY_SECONDS (120): one LIST ACTIVE per name prefix gives every
group's newest article, and new articles become ranges at priority -1,
leased and prefetched before any history. A group gets a tail range once
it has TAIL_MIN_ARTICLES (100k, a chunk) new or its last planning is
TAIL_MAX_DELAY_MINUTES (15) ago, so quiet groups don't make a range every
round. History (planAll, hourly) is now only for groups not planned yet.
ranges:plan --tail plans one round.

Found on the way: with XFEATURE COMPRESS GZIP on, Eweka compresses LIST
ACTIVE's reply too ("215 NewsGroups Follow [COMPRESS=GZIP]"), and
listActive waited for a plain end that never came (groups:add --top too).
It reads the compressed reply now; the fake server in the test does both.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/planner.ts

@@ -54,8 +54,9 @@ class Planner {
     return rows.length
   }
 
+  /** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
   async planAll(): Promise<{ groups: number; ranges: number }> {
-    const groups = await db.from('groups').where('active', true).orderBy('priority').select('id', 'name')
+    const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
     let ranges = 0
     for (const group of groups) {
       // One group failing (e.g. gone from the server) doesn't stop the others.
@@ -67,6 +68,78 @@ class Planner {
     }
     return { groups: groups.length, ranges }
   }
+
+  /**
+   * The live tail: ranges for the articles after each planned group's last
+   * range, at the tail's priority (before any history). The newest article
+   * numbers come from LIST ACTIVE, one command per name prefix (GROUP per
+   * group took minutes for 1000 groups).
+   */
+  async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
+    const groups = await db.from('groups').where('active', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
+    if (groups.length === 0) return { groups: 0, ranges: 0 }
+    const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
+    const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
+    const highs = new Map<string, number>()
+    await NntpPool.use(async (client) => {
+      for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
+        for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
+      }
+    })
+    const rows = tailRanges(
+      groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
+      now,
+      { rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
+    )
+    if (rows.length === 0) return { groups: 0, ranges: 0 }
+    await db.transaction(async (trx) => {
+      await trx.table('ranges').multiInsert(rows)
+      for (const id of new Set(rows.map((r) => r.group_id))) {
+        const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
+        await trx.from('groups').where('id', id).update({ high, planned_at: now })
+      }
+    })
+    return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
+  }
+}
+
+/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
+export function wildmatOf(name: string): string {
+  const parts = name.split('.')
+  return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
+}
+
+export interface TailGroup {
+  id: number
+  /** End of its last range (null: none, then it has no tail yet). */
+  plannedTo: number | null
+  /** When its last range was planned. */
+  plannedAt: Date
+  /** The server's newest article (null: not listed). */
+  high: number | null
+}
+
+/**
+ * The tail's new ranges: per group, the articles after its last range, in
+ * ranges of at most `rangeArticles`, once there are `minArticles` of them or
+ * the group's last planning is `maxDelayMinutes` ago.
+ */
+export function tailRanges(
+  groups: TailGroup[],
+  now: Date,
+  options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
+) {
+  const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
+  for (const g of groups) {
+    if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
+    const waiting = g.high - g.plannedTo
+    const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
+    if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
+    for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
+      rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
+    }
+  }
+  return rows
 }
 
 export default new Planner()