Final completeness by time window: ranges keep their posting span, release_final() over blocks of done ranges
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3d24db5565
commit
ce43099188
12 files changed
+259
-9
No files matched your search
@@ -77,6 +77,13 @@ export interface EdgeBatch {
|
||||
/** First article number not covered yet. */
|
||||
next: number
|
||||
headers: number
|
||||
/**
|
||||
* The batch's posting times without its earliest and latest 1% (ms; null without a dated article): the range's
|
||||
* time span is their min and max (headers dated years off don't stretch it). The settler's final completeness
|
||||
* reads it (LOG.md 2026-10-16 (67)).
|
||||
*/
|
||||
postedLow: number | null
|
||||
postedHigh: number | null
|
||||
releases: ReleaseChunk[]
|
||||
samples: DroppedChunk[]
|
||||
candidates: ProbeCandidate[]
|
||||
@@ -115,6 +122,8 @@ export class RangeIndexer {
|
||||
private aggregator = new FileAggregator<EdgeSegment>(UPLOAD_GAP_MS)
|
||||
private scanner = new SegmentScanner()
|
||||
private segments: EdgeSegment[] = []
|
||||
/** Posting times of this batch's segments (postedSpan at take). */
|
||||
private dates: number[] = []
|
||||
private rules: RuleSet
|
||||
private batch: EdgeBatch
|
||||
private counts = new Map<string, RuleCount>()
|
||||
@@ -155,8 +164,10 @@ export class RangeIndexer {
|
||||
return
|
||||
}
|
||||
const segment = this.scanner.segment(articleNumber, subject, from, date, messageId, bytes, xref, group)
|
||||
if (segment) this.segments.push(segment)
|
||||
else this.count('obfuscated', '', 1, 0)
|
||||
if (segment) {
|
||||
this.segments.push(segment)
|
||||
if (Number.isFinite(segment.postedAt)) this.dates.push(segment.postedAt)
|
||||
} else this.count('obfuscated', '', 1, 0)
|
||||
}
|
||||
|
||||
private endChunk(next: number) {
|
||||
@@ -180,6 +191,8 @@ export class RangeIndexer {
|
||||
const out = this.batch
|
||||
out.counts = [...this.counts.values()]
|
||||
this.counts.clear()
|
||||
;[out.postedLow, out.postedHigh] = postedSpan(this.dates)
|
||||
this.dates = []
|
||||
this.batch = this.empty(out.next)
|
||||
return out
|
||||
}
|
||||
@@ -290,10 +303,18 @@ export class RangeIndexer {
|
||||
}
|
||||
|
||||
private empty(from: number): EdgeBatch {
|
||||
return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
|
||||
return { from, next: from, headers: 0, postedLow: null, postedHigh: null, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
|
||||
}
|
||||
}
|
||||
|
||||
/** Posting times without the earliest and latest 1% (floor(n / 100) at each end); nulls without any. */
|
||||
export function postedSpan(dates: number[]): [number | null, number | null] {
|
||||
if (dates.length === 0) return [null, null]
|
||||
const sorted = Float64Array.from(dates).sort()
|
||||
const k = Math.floor(sorted.length / 100)
|
||||
return [sorted[k], sorted[sorted.length - 1 - k]]
|
||||
}
|
||||
|
||||
/** The release key; a file without a recognisable release becomes a release of its own. */
|
||||
export function releaseKeyOf(s: EdgeSegment): string {
|
||||
return s.file.release?.key ?? hash('sha1', `file\n${s.fileKey}`)
|
||||
|
||||
Reference in new issue
Block a user