From 2c14315f945865abfb189c7ed922e18dcb043a5c Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 02:17:22 +0200 Subject: [PATCH] Rebuild: fragments only for files an edge cut; lone random articles are per-article MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on two neighbouring boneless ranges (merging their fragments): - 99.8% of fragment files were single articles of a random-named 'file' of >100 parts: a random subject per article, which never completes from headers. They now count as per-article. - A file is kept as a fragment when its first part seen is above 1 and comes within the start window (the idle window: uploads in busy groups spread over minutes, and a 50k-article edge zone is ~30 s of boneless), or when it was still open at the range's end with later parts missing. Anything else missing parts is incomplete and only counted. Per range: 70,308 → 130 fragment files; merged, 47 of the 48 files met complete (57,558 segments, ~8% of the range's kept segments). Co-Authored-By: Claude Opus 5.5 --- lib/edge/range_indexer.ts | 76 ++++++++++++++++++++------- tests/functional/batch_store.spec.ts | 2 +- tests/unit/edge/range_indexer.spec.ts | 17 ++++-- 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 753363b..c285aca 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto' import type { Overview } from '../nntp/index.js' import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js' -import { RuleSet, isRandomFile, type Rules } from '../rules/index.js' +import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js' import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js' import { encodeChunk, type ChunkFile } from '../chunks/index.js' @@ -88,7 +88,12 @@ export interface RangeIndexerOptions { rules: Rules /** Chunks without a new part before an open file is given up (as a fragment near an edge). */ idleChunks?: number - /** Within this many articles of an edge, a cut-off file is kept as fragments. */ + /** + * A file whose first part seen is above 1 and comes within this many + * articles of the range's start is kept as a fragment (its earlier parts + * are in the previous range). Default: the idle window (idleChunks of the + * chunks seen so far), as uploads in busy groups spread over minutes. + */ edgeArticles?: number } @@ -109,11 +114,13 @@ export class RangeIndexer { private counts = new Map() private readonly idle: number private readonly edge: number + /** Articles in the first chunk (for the default start window). */ + private chunkArticles = 0 constructor(private options: RangeIndexerOptions) { this.rules = new RuleSet(options.rules) this.idle = options.idleChunks ?? 5 - this.edge = options.edgeArticles ?? 50_000 + this.edge = options.edgeArticles ?? 0 this.batch = this.empty(options.resumeAt ?? options.start) } @@ -149,13 +156,14 @@ export class RangeIndexer { private endChunk(next: number) { this.files(this.aggregator.add(this.segments)) this.segments = [] - this.cutOff(this.aggregator.evict(this.idle)) + if (this.chunkArticles === 0) this.chunkArticles = next - this.batch.from + this.cutOff(this.aggregator.evict(this.idle), false) this.batch.next = next } /** The end of the range: every open file is given up. */ finish() { - this.cutOff(this.aggregator.evict(0)) + this.cutOff(this.aggregator.evict(0), true) this.batch.next = this.options.end + 1 } @@ -205,25 +213,53 @@ export class RangeIndexer { } } - /** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */ - private cutOff(segments: EdgeSegment[]) { - const { start, end } = this.options - const byFile = new Map() + /** + * Given-up files. Kept as fragments only when the range's edge can have + * cut them: parts missing before the first one seen, which comes within + * the start window, or after the last one seen while the file was still + * open at the range's end. Any other file missing parts is incomplete on + * the server, and only counted (in the first 31 ranges, 390k of 406k + * fragment files never met a neighbour; LOG.md 2026-10-09). + */ + private cutOff(segments: EdgeSegment[], atEnd: boolean) { + const { start } = this.options + const window = this.edge || this.idle * Math.max(1, this.chunkArticles) + const byFile = new Map() for (const s of segments) { - const n = Number(s.articleNumber) - if (!Number.isFinite(n) || (n >= start + this.edge && n <= end - this.edge)) { - this.count('incomplete', '', 1, 0) + const list = byFile.get(s.fileKey) + if (list) list.push(s) + else byFile.set(s.fileKey, [s]) + } + for (const list of byFile.values()) { + // One article of a "file" of many parts under a random name: a random subject per article, which + // never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09). + if (list.length === 1 && isRandomFile(list[0].file.name)) { + this.count('per-article', uploadTool(list[0].messageId), 1, 0) continue } - let fragment = byFile.get(s.fileKey) - if (!fragment) { - fragment = fragmentOf(s) - byFile.set(s.fileKey, fragment) - this.batch.fragments.push(fragment) + let minPart = Number.POSITIVE_INFINITY + let maxPart = 0 + let firstArticle = Number.POSITIVE_INFINITY + let lastArticle = 0 + for (const s of list) { + if (s.number < minPart) minPart = s.number + if (s.number > maxPart) maxPart = s.number + if (s.articleNumber < firstArticle) firstArticle = s.articleNumber + if (s.articleNumber > lastArticle) lastArticle = s.articleNumber } - fragment.segments.push([s.number, s.messageId, s.bytes, n]) - const t = s.postedAt - if (Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t + // Cut at the end: still open when the range ends (it had parts within the idle window), later parts missing. + const cutAtStart = minPart > 1 && firstArticle < start + window + const cutAtEnd = atEnd && maxPart < list[0].totalParts + if (!cutAtStart && !cutAtEnd) { + this.count('incomplete', '', list.length, 1) + continue + } + const fragment = fragmentOf(list[0]) + for (const s of list) { + fragment.segments.push([s.number, s.messageId, s.bytes, s.articleNumber]) + if (Number.isFinite(s.postedAt) && (fragment.postedAt === null || s.postedAt < fragment.postedAt)) fragment.postedAt = s.postedAt + } + this.batch.fragments.push(fragment) } } diff --git a/tests/functional/batch_store.spec.ts b/tests/functional/batch_store.spec.ts index 55a0ee8..797b711 100644 --- a/tests/functional/batch_store.spec.ts +++ b/tests/functional/batch_store.spec.ts @@ -42,7 +42,7 @@ test.group('BatchStore', (group) => { ...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 2, 'Some.Show.S01E01 [1/2]'), ...post(100_010, 'Some.Show.S01E01.1080p.WEB.par2', 1, 'Some.Show.S01E01 [2/2]'), ...post(100_020, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 3, '[1/1]', 'ngPost'), - ...post(10, 'Edge.Cut.2020.part1.rar', 3, 'Edge.Cut [1/1]').slice(0, 1), + ...post(10, 'Edge.Cut.2020.part1.rar', 3, 'Edge.Cut [1/1]').slice(1), ], 500_000) const first = ix.take() const stored = await BatchStore.store(rangeId, nodeId, first) diff --git a/tests/unit/edge/range_indexer.spec.ts b/tests/unit/edge/range_indexer.spec.ts index be7d13a..efab6e5 100644 --- a/tests/unit/edge/range_indexer.spec.ts +++ b/tests/unit/edge/range_indexer.spec.ts @@ -55,13 +55,22 @@ test.group('edge / RangeIndexer', () => { assert.equal(batch.next, 1_000_001) }) - test('a file cut off near an edge leaves as fragments, one in the middle is only counted', ({ assert }) => { + test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => { const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 }) - indexer.add([...post(5, 'Edge.File.2020.part1.rar', 3).slice(0, 2), ...post(500_000, 'Middle.File.2020.part1.rar', 3).slice(0, 2)], 600_000) + indexer.add([ + // Parts 2–3 right after the start: part 1 is in the previous range. + ...post(5, 'Start.Cut.2020.part1.rar', 3).slice(1), + // Parts 1–2 seen, part 3 never: incomplete on the server. + ...post(500_000, 'Middle.Hole.2020.part1.rar', 3).slice(0, 2), + ], 600_000) + // Chunks without its part 3: the hole file is given up (idle) long before the end. + for (let at = 650_000; at < 950_000; at += 50_000) indexer.add([], at) + // Parts 1–2 near the end, still open when the range ends: part 3 is in the next range. + indexer.add(post(999_990, 'End.Cut.2020.part1.rar', 3).slice(0, 2), 1_000_001) indexer.finish() const batch = indexer.take() - assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => [s[0], s[3]])]), [['Edge.File.2020.part1.rar', [[1, 5], [2, 6]]]]) - assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles]), [['incomplete', 2]]) + assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => s[0])]).sort(), [['End.Cut.2020.part1.rar', [1, 2]], ['Start.Cut.2020.part1.rar', [2, 3]]]) + assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]]) assert.lengthOf(batch.releases, 0) }) })