usenet-indexer/app/services/group_dates.ts

import type { Overview, XoverClient } from '#lib/nntp/index'

/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
export interface DateSample {
  article: number
  postedAt: Date
}

/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
const OLDEST = Date.UTC(1995, 0, 1)
const DAY = 86_400_000

/**
 * The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
 * `min` of them; null otherwise (an expired or empty slice).
 */
export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
  const latest = now.getTime() + DAY
  const times = overviews
    .map(([, fields]) => Date.parse(fields.date ?? ''))
    .filter((t) => t >= OLDEST && t <= latest)
    .sort((a, b) => a - b)
  if (times.length < min) return null
  return new Date(times[Math.floor(times.length / 2)])
}

/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
export function samplePoints(low: number, high: number, count: number, size: number): number[] {
  const last = Math.max(low, high - size + 1)
  if (count <= 1 || last <= low) return [low]
  const points = new Set<number>()
  for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
  return [...points].sort((a, b) => a - b)
}

/**
 * The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
 * (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
 * in number order, bar stragglers and old crossposts).
 */
export function monotone(samples: DateSample[]): DateSample[] {
  const sorted = [...samples].sort((a, b) => a.article - b.article)
  const kept = sorted.filter((s, i) => {
    const next = sorted.slice(i + 1, i + 3)
    return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
  })
  let floor = Number.NEGATIVE_INFINITY
  return kept.map((s) => {
    floor = Math.max(floor, s.postedAt.getTime())
    return { article: s.article, postedAt: new Date(floor) }
  })
}

/**
 * When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
 * it, the first or last sample's date beyond them; null without samples.
 */
export function estimateDate(samples: DateSample[], article: number): Date | null {
  if (samples.length === 0) return null
  if (article <= samples[0].article) return samples[0].postedAt
  const last = samples[samples.length - 1]
  if (article >= last.article) return last.postedAt
  let lo = 0
  let hi = samples.length - 1
  while (hi - lo > 1) {
    const mid = (lo + hi) >> 1
    if (samples[mid].article <= article) lo = mid
    else hi = mid
  }
  const a = samples[lo]
  const b = samples[hi]
  const f = (article - a.article) / (b.article - a.article)
  return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
}

/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
export function sampleCount(
  low: number,
  high: number,
  every: number,
  min = 16,
  max = 1000
): number {
  return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
}

/**
 * Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
 * with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
 * server's span.
 */
export async function sampleGroup(
  use: <T>(work: (client: XoverClient) => Promise<T>) => Promise<T>,
  name: string,
  options: { every: number; size: number; maxFailures?: number },
  now = new Date()
): Promise<{
  samples: DateSample[]
  low: number
  high: number
  commands: number
  failed: string[]
}> {
  const info = await use((client) => client.group(name))
  const samples: DateSample[] = []
  const failed: string[] = []
  let commands = 1
  // Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for
  // minutes; 16 groups never got through on one connection) costs that point, not the group.
  for (const point of samplePoints(
    info.low,
    info.high,
    sampleCount(info.low, info.high, options.every),
    options.size
  )) {
    if (failed.length >= (options.maxFailures ?? 3)) break
    try {
      const date = await use(async (client) => {
        await client.group(name)
        commands += 2
        const slice = medianDate(
          await client.xover(point, Math.min(point + options.size - 1, info.high)),
          now
        )
        if (slice) return slice
        commands++
        return medianDate(
          await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
          now
        )
      })
      if (date) samples.push({ article: point, postedAt: date })
    } catch (error: any) {
      failed.push(`${point}: ${error.message}`)
    }
  }
  return { samples: monotone(samples), low: info.low, high: info.high, commands, failed }
}