The tail planner grows a group's open range; ranges:merge-tail packs the slivers

A round added a row a group whenever 100k articles or 15 minutes had passed,
tuned for a tail leased within minutes; with nothing leasing the tail the rows
piled up (127k planned tail ranges of a median 14k articles, 97% adjacent). Now
a round grows the group's last range when it is a planned, unspooled, untaken
row at the tail's priority, up to RANGE_ARTICLES, and only then adds rows; a
range taken meanwhile gets the slice as a row. ranges:merge-tail (--dry-run)
packs adjacent planned, uncaptured, unspooled, untaken tail ranges of a group
and priority into ranges of RANGE_ARTICLES under the lease lock.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

commands/coordinator_run.ts

@@ -117,8 +117,11 @@ export default class CoordinatorRun extends BaseCommand {
         }
         if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
           tailedAt = Date.now()
-          const { groups, ranges } = await Planner.planTail()
-          if (ranges > 0) this.logger.info(`Tail: ${ranges} ranges for ${groups} groups.`)
+          const { groups, ranges, extended } = await Planner.planTail()
+          if (ranges + extended > 0)
+            this.logger.info(
+              `Tail: ${ranges} ranges added, ${extended} grown, for ${groups} groups.`
+            )
         }
       } catch (error: any) {
         this.logger.error(`Coordinator round failed: ${error.message}`)

commands/ranges_merge_tail.ts

@@ -0,0 +1,23 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import Planner from '#services/planner'
+
+/**
+ * Merges adjacent planned tail ranges (Planner.mergeTail): the tail planner added a sliver a group every round
+ * while nothing leased the tail; now it grows the group's open range instead, and this packs what is there.
+ */
+export default class RangesMergeTail extends BaseCommand {
+  static commandName = 'ranges:merge-tail'
+  static description = 'Merges adjacent planned tail ranges into ranges of RANGE_ARTICLES'
+  static options: CommandOptions = { startApp: true }
+
+  @flags.boolean({ description: 'Only count what would be merged' })
+  declare dryRun: boolean
+
+  async run() {
+    const { candidates, merged, dropped } = await Planner.mergeTail(this.dryRun)
+    this.logger.info(
+      `${candidates} planned tail ranges: ${dropped} ${this.dryRun ? 'would merge' : 'merged'} into ${merged} grown ranges (${candidates - dropped} left).`
+    )
+  }
+}

commands/ranges_plan.ts

@@ -14,8 +14,13 @@ export default class RangesPlan extends BaseCommand {
 
   async run() {
     try {
-      const { groups, ranges } = this.tail ? await Planner.planTail() : await Planner.planAll()
-      this.logger.info(`${ranges} ${this.tail ? 'tail ' : ''}ranges planned over ${groups} groups.`)
+      if (this.tail) {
+        const { groups, ranges, extended } = await Planner.planTail()
+        this.logger.info(`Tail: ${ranges} ranges added, ${extended} grown, over ${groups} groups.`)
+      } else {
+        const { groups, ranges } = await Planner.planAll()
+        this.logger.info(`${ranges} ranges planned over ${groups} groups.`)
+      }
     } finally {
       await NntpPool.close()
     }