Final completeness by time window: ranges keep their posting span, release_final() over blocks of done ranges
Test and deploy / test (push) Successful in 58s
Test and deploy / edge (push) Successful in 2m4s
Test and deploy / deploy (push) Successful in 8m5s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-16 14:19:03 +02:00
1 parent 3d24db5565
commit ce43099188
12 files changed
+259 -9

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
export { RangeIndexer, chunkOf, candidatesOf, releaseKeyOf, uploadsOf, UPLOAD_GAP_MS, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
export { RangeIndexer, postedSpan, chunkOf, candidatesOf, releaseKeyOf, uploadsOf, UPLOAD_GAP_MS, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
export { toWire, fromWire, type WireBatch } from './wire.js'
export type { Lease } from './lease.js'
+24 -3
View File
@@ -77,6 +77,13 @@ export interface EdgeBatch {
/** First article number not covered yet. */
next: number
headers: number
/**
* The batch's posting times without its earliest and latest 1% (ms; null without a dated article): the range's
* time span is their min and max (headers dated years off don't stretch it). The settler's final completeness
* reads it (LOG.md 2026-10-16 (67)).
*/
postedLow: number | null
postedHigh: number | null
releases: ReleaseChunk[]
samples: DroppedChunk[]
candidates: ProbeCandidate[]
@@ -115,6 +122,8 @@ export class RangeIndexer {
private aggregator = new FileAggregator<EdgeSegment>(UPLOAD_GAP_MS)
private scanner = new SegmentScanner()
private segments: EdgeSegment[] = []
/** Posting times of this batch's segments (postedSpan at take). */
private dates: number[] = []
private rules: RuleSet
private batch: EdgeBatch
private counts = new Map<string, RuleCount>()
@@ -155,8 +164,10 @@ export class RangeIndexer {
return
}
const segment = this.scanner.segment(articleNumber, subject, from, date, messageId, bytes, xref, group)
if (segment) this.segments.push(segment)
else this.count('obfuscated', '', 1, 0)
if (segment) {
this.segments.push(segment)
if (Number.isFinite(segment.postedAt)) this.dates.push(segment.postedAt)
} else this.count('obfuscated', '', 1, 0)
}
private endChunk(next: number) {
@@ -180,6 +191,8 @@ export class RangeIndexer {
const out = this.batch
out.counts = [...this.counts.values()]
this.counts.clear()
;[out.postedLow, out.postedHigh] = postedSpan(this.dates)
this.dates = []
this.batch = this.empty(out.next)
return out
}
@@ -290,10 +303,18 @@ export class RangeIndexer {
}
private empty(from: number): EdgeBatch {
return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
return { from, next: from, headers: 0, postedLow: null, postedHigh: null, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
}
}
/** Posting times without the earliest and latest 1% (floor(n / 100) at each end); nulls without any. */
export function postedSpan(dates: number[]): [number | null, number | null] {
if (dates.length === 0) return [null, null]
const sorted = Float64Array.from(dates).sort()
const k = Math.floor(sorted.length / 100)
return [sorted[k], sorted[sorted.length - 1 - k]]
}
/** The release key; a file without a recognisable release becomes a release of its own. */
export function releaseKeyOf(s: EdgeSegment): string {
return s.file.release?.key ?? hash('sha1', `file\n${s.fileKey}`)