import type { Overview, XoverClient } from '#lib/nntp/index' /** When an article of a group was posted: the median Date of a slice of articles at `article`. */ export interface DateSample { article: number postedAt: Date } /** Before this, a Date header is a broken clock (the oldest retention is 2008). */ const OLDEST = Date.UTC(1995, 0, 1) const DAY = 86_400_000 /** * The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least * `min` of them; null otherwise (an expired or empty slice). */ export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null { const latest = now.getTime() + DAY const times = overviews .map(([, fields]) => Date.parse(fields.date ?? '')) .filter((t) => t >= OLDEST && t <= latest) .sort((a, b) => a - b) if (times.length < min) return null return new Date(times[Math.floor(times.length / 2)]) } /** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */ export function samplePoints(low: number, high: number, count: number, size: number): number[] { const last = Math.max(low, high - size + 1) if (count <= 1 || last <= low) return [low] const points = new Set() for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1))) return [...points].sort((a, b) => a - b) } /** * The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day * (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive * in number order, bar stragglers and old crossposts). */ export function monotone(samples: DateSample[]): DateSample[] { const sorted = [...samples].sort((a, b) => a.article - b.article) const kept = sorted.filter((s, i) => { const next = sorted.slice(i + 1, i + 3) return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime()) }) let floor = Number.NEGATIVE_INFINITY return kept.map((s) => { floor = Math.max(floor, s.postedAt.getTime()) return { article: s.article, postedAt: new Date(floor) } }) } /** * When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around * it, the first or last sample's date beyond them; null without samples. */ export function estimateDate(samples: DateSample[], article: number): Date | null { if (samples.length === 0) return null if (article <= samples[0].article) return samples[0].postedAt const last = samples[samples.length - 1] if (article >= last.article) return last.postedAt let lo = 0 let hi = samples.length - 1 while (hi - lo > 1) { const mid = (lo + hi) >> 1 if (samples[mid].article <= article) lo = mid else hi = mid } const a = samples[lo] const b = samples[hi] const f = (article - a.article) / (b.article - a.article) return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime())) } /** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */ export function sampleCount( low: number, high: number, every: number, min = 16, max = 1000 ): number { return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1)) } /** * Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice * with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the * server's span. */ export async function sampleGroup( use: (work: (client: XoverClient) => Promise) => Promise, name: string, options: { every: number; size: number; maxFailures?: number }, now = new Date() ): Promise<{ samples: DateSample[] low: number high: number commands: number failed: string[] }> { const info = await use((client) => client.group(name)) const samples: DateSample[] = [] const failed: string[] = [] let commands = 1 // Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for // minutes; 16 groups never got through on one connection) costs that point, not the group. for (const point of samplePoints( info.low, info.high, sampleCount(info.low, info.high, options.every), options.size )) { if (failed.length >= (options.maxFailures ?? 3)) break try { const date = await use(async (client) => { await client.group(name) commands += 2 const slice = medianDate( await client.xover(point, Math.min(point + options.size - 1, info.high)), now ) if (slice) return slice commands++ return medianDate( await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)), now ) }) if (date) samples.push({ article: point, postedAt: date }) } catch (error: any) { failed.push(`${point}: ${error.message}`) } } return { samples: monotone(samples), low: info.low, high: info.high, commands, failed } }