Rebuild: fragments only for files an edge cut; lone random articles are per-article

Measured on two neighbouring boneless ranges (merging their fragments):
- 99.8% of fragment files were single articles of a random-named
  'file' of >100 parts: a random subject per article, which never
  completes from headers. They now count as per-article.
- A file is kept as a fragment when its first part seen is above 1 and
  comes within the start window (the idle window: uploads in busy groups
  spread over minutes, and a 50k-article edge zone is ~30 s of boneless),
  or when it was still open at the range's end with later parts missing.
  Anything else missing parts is incomplete and only counted.
Per range: 70,308 → 130 fragment files; merged, 47 of the 48 files met
complete (57,558 segments, ~8% of the range's kept segments).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/edge/range_indexer.ts

@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto'
 import type { Overview } from '../nntp/index.js'
 import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
-import { RuleSet, isRandomFile, type Rules } from '../rules/index.js'
+import { RuleSet, isRandomFile, uploadTool, type Rules } from '../rules/index.js'
 import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
 import { encodeChunk, type ChunkFile } from '../chunks/index.js'
 
@@ -88,7 +88,12 @@ export interface RangeIndexerOptions {
   rules: Rules
   /** Chunks without a new part before an open file is given up (as a fragment near an edge). */
   idleChunks?: number
-  /** Within this many articles of an edge, a cut-off file is kept as fragments. */
+  /**
+   * A file whose first part seen is above 1 and comes within this many
+   * articles of the range's start is kept as a fragment (its earlier parts
+   * are in the previous range). Default: the idle window (idleChunks of the
+   * chunks seen so far), as uploads in busy groups spread over minutes.
+   */
   edgeArticles?: number
 }
 
@@ -109,11 +114,13 @@ export class RangeIndexer {
   private counts = new Map<string, RuleCount>()
   private readonly idle: number
   private readonly edge: number
+  /** Articles in the first chunk (for the default start window). */
+  private chunkArticles = 0
 
   constructor(private options: RangeIndexerOptions) {
     this.rules = new RuleSet(options.rules)
     this.idle = options.idleChunks ?? 5
-    this.edge = options.edgeArticles ?? 50_000
+    this.edge = options.edgeArticles ?? 0
     this.batch = this.empty(options.resumeAt ?? options.start)
   }
 
@@ -149,13 +156,14 @@ export class RangeIndexer {
   private endChunk(next: number) {
     this.files(this.aggregator.add(this.segments))
     this.segments = []
-    this.cutOff(this.aggregator.evict(this.idle))
+    if (this.chunkArticles === 0) this.chunkArticles = next - this.batch.from
+    this.cutOff(this.aggregator.evict(this.idle), false)
     this.batch.next = next
   }
 
   /** The end of the range: every open file is given up. */
   finish() {
-    this.cutOff(this.aggregator.evict(0))
+    this.cutOff(this.aggregator.evict(0), true)
     this.batch.next = this.options.end + 1
   }
 
@@ -205,25 +213,53 @@ export class RangeIndexer {
     }
   }
 
-  /** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */
-  private cutOff(segments: EdgeSegment[]) {
-    const { start, end } = this.options
-    const byFile = new Map<string, Fragment>()
+  /**
+   * Given-up files. Kept as fragments only when the range's edge can have
+   * cut them: parts missing before the first one seen, which comes within
+   * the start window, or after the last one seen while the file was still
+   * open at the range's end. Any other file missing parts is incomplete on
+   * the server, and only counted (in the first 31 ranges, 390k of 406k
+   * fragment files never met a neighbour; LOG.md 2026-10-09).
+   */
+  private cutOff(segments: EdgeSegment[], atEnd: boolean) {
+    const { start } = this.options
+    const window = this.edge || this.idle * Math.max(1, this.chunkArticles)
+    const byFile = new Map<string, EdgeSegment[]>()
     for (const s of segments) {
-      const n = Number(s.articleNumber)
-      if (!Number.isFinite(n) || (n >= start + this.edge && n <= end - this.edge)) {
-        this.count('incomplete', '', 1, 0)
+      const list = byFile.get(s.fileKey)
+      if (list) list.push(s)
+      else byFile.set(s.fileKey, [s])
+    }
+    for (const list of byFile.values()) {
+      // One article of a "file" of many parts under a random name: a random subject per article, which
+      // never completes from headers (99.8% of the boneless fragments; LOG.md 2026-10-09).
+      if (list.length === 1 && isRandomFile(list[0].file.name)) {
+        this.count('per-article', uploadTool(list[0].messageId), 1, 0)
         continue
       }
-      let fragment = byFile.get(s.fileKey)
-      if (!fragment) {
-        fragment = fragmentOf(s)
-        byFile.set(s.fileKey, fragment)
-        this.batch.fragments.push(fragment)
+      let minPart = Number.POSITIVE_INFINITY
+      let maxPart = 0
+      let firstArticle = Number.POSITIVE_INFINITY
+      let lastArticle = 0
+      for (const s of list) {
+        if (s.number < minPart) minPart = s.number
+        if (s.number > maxPart) maxPart = s.number
+        if (s.articleNumber < firstArticle) firstArticle = s.articleNumber
+        if (s.articleNumber > lastArticle) lastArticle = s.articleNumber
       }
-      fragment.segments.push([s.number, s.messageId, s.bytes, n])
-      const t = s.postedAt
-      if (Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
+      // Cut at the end: still open when the range ends (it had parts within the idle window), later parts missing.
+      const cutAtStart = minPart > 1 && firstArticle < start + window
+      const cutAtEnd = atEnd && maxPart < list[0].totalParts
+      if (!cutAtStart && !cutAtEnd) {
+        this.count('incomplete', '', list.length, 1)
+        continue
+      }
+      const fragment = fragmentOf(list[0])
+      for (const s of list) {
+        fragment.segments.push([s.number, s.messageId, s.bytes, s.articleNumber])
+        if (Number.isFinite(s.postedAt) && (fragment.postedAt === null || s.postedAt < fragment.postedAt)) fragment.postedAt = s.postedAt
+      }
+      this.batch.fragments.push(fragment)
     }
   }
 

tests/functional/batch_store.spec.ts

@@ -42,7 +42,7 @@ test.group('BatchStore', (group) => {
       ...post(100_000, 'Some.Show.S01E01.1080p.WEB.part1.rar', 2, 'Some.Show.S01E01 [1/2]'),
       ...post(100_010, 'Some.Show.S01E01.1080p.WEB.par2', 1, 'Some.Show.S01E01 [2/2]'),
       ...post(100_020, 'Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 3, '[1/1]', 'ngPost'),
-      ...post(10, 'Edge.Cut.2020.part1.rar', 3, 'Edge.Cut [1/1]').slice(0, 1),
+      ...post(10, 'Edge.Cut.2020.part1.rar', 3, 'Edge.Cut [1/1]').slice(1),
     ], 500_000)
     const first = ix.take()
     const stored = await BatchStore.store(rangeId, nodeId, first)

tests/unit/edge/range_indexer.spec.ts

@@ -55,13 +55,22 @@ test.group('edge / RangeIndexer', () => {
     assert.equal(batch.next, 1_000_001)
   })
 
-  test('a file cut off near an edge leaves as fragments, one in the middle is only counted', ({ assert }) => {
+  test('files the edges cut leave as fragments; one missing parts in the middle is only counted', ({ assert }) => {
     const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1_000_000, rules, edgeArticles: 1000 })
-    indexer.add([...post(5, 'Edge.File.2020.part1.rar', 3).slice(0, 2), ...post(500_000, 'Middle.File.2020.part1.rar', 3).slice(0, 2)], 600_000)
+    indexer.add([
+      // Parts 2–3 right after the start: part 1 is in the previous range.
+      ...post(5, 'Start.Cut.2020.part1.rar', 3).slice(1),
+      // Parts 1–2 seen, part 3 never: incomplete on the server.
+      ...post(500_000, 'Middle.Hole.2020.part1.rar', 3).slice(0, 2),
+    ], 600_000)
+    // Chunks without its part 3: the hole file is given up (idle) long before the end.
+    for (let at = 650_000; at < 950_000; at += 50_000) indexer.add([], at)
+    // Parts 1–2 near the end, still open when the range ends: part 3 is in the next range.
+    indexer.add(post(999_990, 'End.Cut.2020.part1.rar', 3).slice(0, 2), 1_000_001)
     indexer.finish()
     const batch = indexer.take()
-    assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => [s[0], s[3]])]), [['Edge.File.2020.part1.rar', [[1, 5], [2, 6]]]])
-    assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles]), [['incomplete', 2]])
+    assert.deepEqual(batch.fragments.map((f) => [f.name, f.segments.map((s) => s[0])]).sort(), [['End.Cut.2020.part1.rar', [1, 2]], ['Start.Cut.2020.part1.rar', [2, 3]]])
+    assert.deepEqual(batch.counts.map((c) => [c.rule, c.articles, c.files]), [['incomplete', 2, 1]])
     assert.lengthOf(batch.releases, 0)
   })
 })