Posting-time windows into the corpus: ranges:dates, ranges:capture

ranges:dates samples each active group's posting dates from the news
server (group_dates) and estimates every range's date (ranges.est_date).
ranges:capture --from --to marks a window's ranges of every group for the
corpus. The corpus fill takes capture ranges in priority order, done ones
too, and remembers complete ones (ranges.stored_at).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/services/group_dates.ts

@@ -0,0 +1,122 @@
+import type { Overview, XoverClient } from '#lib/nntp/index'
+
+/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
+export interface DateSample {
+  article: number
+  postedAt: Date
+}
+
+/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
+const OLDEST = Date.UTC(1995, 0, 1)
+const DAY = 86_400_000
+
+/**
+ * The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
+ * `min` of them; null otherwise (an expired or empty slice).
+ */
+export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
+  const latest = now.getTime() + DAY
+  const times = overviews
+    .map(([, fields]) => Date.parse(fields.date ?? ''))
+    .filter((t) => t >= OLDEST && t <= latest)
+    .sort((a, b) => a - b)
+  if (times.length < min) return null
+  return new Date(times[Math.floor(times.length / 2)])
+}
+
+/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
+export function samplePoints(low: number, high: number, count: number, size: number): number[] {
+  const last = Math.max(low, high - size + 1)
+  if (count <= 1 || last <= low) return [low]
+  const points = new Set<number>()
+  for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
+  return [...points].sort((a, b) => a - b)
+}
+
+/**
+ * The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
+ * (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
+ * in number order, bar stragglers and old crossposts).
+ */
+export function monotone(samples: DateSample[]): DateSample[] {
+  const sorted = [...samples].sort((a, b) => a.article - b.article)
+  const kept = sorted.filter((s, i) => {
+    const next = sorted.slice(i + 1, i + 3)
+    return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
+  })
+  let floor = Number.NEGATIVE_INFINITY
+  return kept.map((s) => {
+    floor = Math.max(floor, s.postedAt.getTime())
+    return { article: s.article, postedAt: new Date(floor) }
+  })
+}
+
+/**
+ * When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
+ * it, the first or last sample's date beyond them; null without samples.
+ */
+export function estimateDate(samples: DateSample[], article: number): Date | null {
+  if (samples.length === 0) return null
+  if (article <= samples[0].article) return samples[0].postedAt
+  const last = samples[samples.length - 1]
+  if (article >= last.article) return last.postedAt
+  let lo = 0
+  let hi = samples.length - 1
+  while (hi - lo > 1) {
+    const mid = (lo + hi) >> 1
+    if (samples[mid].article <= article) lo = mid
+    else hi = mid
+  }
+  const a = samples[lo]
+  const b = samples[hi]
+  const f = (article - a.article) / (b.article - a.article)
+  return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
+}
+
+/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
+export function sampleCount(
+  low: number,
+  high: number,
+  every: number,
+  min = 16,
+  max = 1000
+): number {
+  return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
+}
+
+/**
+ * Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
+ * with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
+ * server's span.
+ */
+export async function sampleGroup(
+  client: XoverClient,
+  name: string,
+  options: { every: number; size: number },
+  now = new Date()
+): Promise<{ samples: DateSample[]; low: number; high: number; commands: number }> {
+  const info = await client.group(name)
+  const samples: DateSample[] = []
+  let commands = 1
+  for (const point of samplePoints(
+    info.low,
+    info.high,
+    sampleCount(info.low, info.high, options.every),
+    options.size
+  )) {
+    commands++
+    let date = medianDate(
+      await client.xover(point, Math.min(point + options.size - 1, info.high)),
+      now
+    )
+    if (!date) {
+      commands++
+      date = medianDate(
+        await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
+        now
+      )
+    }
+    if (date) samples.push({ article: point, postedAt: date })
+  }
+  return { samples: monotone(samples), low: info.low, high: info.high, commands }
+}