Rebuild: fragments only for files an edge cut; lone random articles are per-article

Measured on two neighbouring boneless ranges (merging their fragments):
- 99.8% of fragment files were single articles of a random-named
  'file' of >100 parts: a random subject per article, which never
  completes from headers. They now count as per-article.
- A file is kept as a fragment when its first part seen is above 1 and
  comes within the start window (the idle window: uploads in busy groups
  spread over minutes, and a 50k-article edge zone is ~30 s of boneless),
  or when it was still open at the range's end with later parts missing.
  Anything else missing parts is incomplete and only counted.
Per range: 70,308 → 130 fragment files; merged, 47 of the 48 files met
complete (57,558 segments, ~8% of the range's kept segments).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/edge/range_indexer.ts

@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto'
 import type { Overview } from '../nntp/index.js'
 import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
-import { RuleSet, isRandomFile, type Rules } from '../rules/index.js'
+import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js'
 import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
 import { encodeChunk, type ChunkFile } from '../chunks/index.js'
 
@@ -88,7 +88,12 @@ export interface RangeIndexerOptions {
   rules: Rules
   /** Chunks without a new part before an open file is given up (as a fragment near an edge). */
   idleChunks?: number
-  /** Within this many articles of an edge, a cut-off file is kept as fragments. */
+  /**
+   * A file whose first part seen is above 1 and comes within this many
+   * articles of the range's start is kept as a fragment (its earlier parts
+   * are in the previous range). Default: the idle window (idleChunks of the
+   * chunks seen so far), as uploads in busy groups spread over minutes.
+   */
   edgeArticles?: number
 }
 
@@ -109,11 +114,13 @@ export class RangeIndexer {
   private counts = new Map<string, RuleCount>()
   private readonly idle: number
   private readonly edge: number
+  /** Articles in the first chunk (for the default start window). */
+  private chunkArticles = 0
 
   constructor(private options: RangeIndexerOptions) {
     this.rules = new RuleSet(options.rules)
     this.idle = options.idleChunks ?? 5
-    this.edge = options.edgeArticles ?? 50_000
+    this.edge = options.edgeArticles ?? 0
     this.batch = this.empty(options.resumeAt ?? options.start)
   }
 
@@ -149,13 +156,14 @@ export class RangeIndexer {
   private endChunk(next: number) {
     this.files(this.aggregator.add(this.segments))
     this.segments = []
-    this.cutOff(this.aggregator.evict(this.idle))
+    if (this.chunkArticles === 0) this.chunkArticles = next - this.batch.from
+    this.cutOff(this.aggregator.evict(this.idle), false)
     this.batch.next = next
   }
 
   /** The end of the range: every open file is given up. */
   finish() {
-    this.cutOff(this.aggregator.evict(0))
+    this.cutOff(this.aggregator.evict(0), true)
     this.batch.next = this.options.end + 1
   }
 
@@ -205,25 +213,53 @@ export class RangeIndexer {
     }
   }
 
-  /** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */
-  private cutOff(segments: EdgeSegment[]) {
-    const { start, end } = this.options
-    const byFile = new Map<string, Fragment>()
+  /**
+   * Given-up files. Kept as fragments only when the range's edge can have
+   * cut them: parts missing before the first one seen, which comes within
+   * the start window, or after the last one seen while the file was still
+   * open at the range's end. Any other file missing parts is incomplete on
+   * the server, and only counted (in the first 31 ranges, 390k of 406k
+   * fragment files never met a neighbour; LOG.md 2026-10-09).
+   */
+  private cutOff(segments: EdgeSegment[], atEnd: boolean) {
+    const { start } = this.options
+    const window = this.edge || this.idle * Math.max(1, this.chunkArticles)
+    const byFile = new Map<string, EdgeSegment[]>()
     for (const s of segments) {
-      const n = Number(s.articleNumber)
-      if (!Number.isFinite(n) || (n >= start + this.edge && n <= end - this.edge)) {
-        this.count('incomplete', '', 1, 0)
+      const list = byFile.get(s.fileKey)
+      if (list) list.push(s)
+      else byFile.set(s.fileKey, [s])
+    }
+    for (const list of byFile.values()) {
+      // One article of a "file" of many parts under a random name: a random subject per article, which
+      // never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09).
+      if (list.length === 1 && isRandomFile(list[0].file.name)) {
+        this.count('per-article', uploadTool(list[0].messageId), 1, 0)
         continue
       }
-      let fragment = byFile.get(s.fileKey)
-      if (!fragment) {
-        fragment = fragmentOf(s)
-        byFile.set(s.fileKey, fragment)
-        this.batch.fragments.push(fragment)
+      let minPart = Number.POSITIVE_INFINITY
+      let maxPart = 0
+      let firstArticle = Number.POSITIVE_INFINITY
+      let lastArticle = 0
+      for (const s of list) {
+        if (s.number < minPart) minPart = s.number
+        if (s.number > maxPart) maxPart = s.number
+        if (s.articleNumber < firstArticle) firstArticle = s.articleNumber
+        if (s.articleNumber > lastArticle) lastArticle = s.articleNumber
       }
-      fragment.segments.push([s.number, s.messageId, s.bytes, n])
-      const t = s.postedAt
-      if (Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
+      // Cut at the end: still open when the range ends (it had parts within the idle window), later parts missing.
+      const cutAtStart = minPart > 1 && firstArticle < start + window
+      const cutAtEnd = atEnd && maxPart < list[0].totalParts
+      if (!cutAtStart && !cutAtEnd) {
+        this.count('incomplete', '', list.length, 1)
+        continue
+      }
+      const fragment = fragmentOf(list[0])
+      for (const s of list) {
+        fragment.segments.push([s.number, s.messageId, s.bytes, s.articleNumber])
+        if (Number.isFinite(s.postedAt) && (fragment.postedAt === null || s.postedAt < fragment.postedAt)) fragment.postedAt = s.postedAt
+      }
+      this.batch.fragments.push(fragment)
     }
   }