The tail planner grows a group's open range; ranges:merge-tail packs the slivers

A round added a row a group whenever 100k articles or 15 minutes had passed,
tuned for a tail leased within minutes; with nothing leasing the tail the rows
piled up (127k planned tail ranges of a median 14k articles, 97% adjacent). Now
a round grows the group's last range when it is a planned, unspooled, untaken
row at the tail's priority, up to RANGE_ARTICLES, and only then adds rows; a
range taken meanwhile gets the slice as a row. ranges:merge-tail (--dry-run)
packs adjacent planned, uncaptured, unspooled, untaken tail ranges of a group
and priority into ranges of RANGE_ARTICLES under the lease lock.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/functional/planner_tail.spec.ts

@@ -0,0 +1,142 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import Planner from '#services/planner'
+import coordinatorConfig from '#config/coordinator'
+
+const TAIL = coordinatorConfig.tail.priority
+
+test.group('Planner / tail ranges', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('applyTail grows an open range, or adds the slice when the range was taken meanwhile', async ({
+    assert,
+  }) => {
+    const [g] = await db
+      .table('groups')
+      .insert({
+        name: 'alt.binaries.tail',
+        priority: 1,
+        tail: true,
+        planned_at: new Date('2026-10-27T08:00:00Z'),
+      })
+      .returning('id')
+    const [open] = await db
+      .table('ranges')
+      .insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, priority: TAIL })
+      .returning('id')
+    const [taken] = await db
+      .table('ranges')
+      .insert({
+        group_id: g.id,
+        start_id: 301,
+        end_id: 400,
+        next_id: 301,
+        priority: TAIL,
+        status: 'leased',
+        node_id: 1,
+      })
+      .returning('id')
+    const now = new Date('2026-10-27T09:00:00Z')
+    const planned = await Planner.applyTail(
+      {
+        rows: [
+          { group_id: Number(g.id), start_id: 2001, end_id: 2500, next_id: 2001, priority: TAIL },
+        ],
+        extended: [
+          { id: Number(open.id), group_id: Number(g.id), from: 301, end_id: 2000 },
+          { id: Number(taken.id), group_id: Number(g.id), from: 401, end_id: 600 },
+        ],
+      },
+      now
+    )
+    assert.deepEqual(planned, { groups: 1, ranges: 2, extended: 1 })
+    const ranges = await db
+      .from('ranges')
+      .where('group_id', g.id)
+      .orderBy('start_id')
+      .select('start_id', 'end_id', 'status', 'priority')
+    assert.deepEqual(
+      ranges.map((r) => [Number(r.start_id), Number(r.end_id), r.status, Number(r.priority)]),
+      [
+        [1, 2000, 'planned', TAIL],
+        [301, 400, 'leased', TAIL],
+        [401, 600, 'planned', TAIL],
+        [2001, 2500, 'planned', TAIL],
+      ]
+    )
+    const updated = await db.from('groups').where('id', g.id).firstOrFail()
+    assert.equal(Number(updated.high), 2500)
+    assert.equal(new Date(updated.planned_at).toISOString(), now.toISOString())
+  })
+
+  test('mergeTail packs adjacent planned tail ranges and leaves taken, spooled, capture and history ranges alone', async ({
+    assert,
+  }) => {
+    const saved = coordinatorConfig.rangeArticles
+    coordinatorConfig.rangeArticles = 1000
+    try {
+      const [g] = await db
+        .table('groups')
+        .insert({ name: 'alt.binaries.tail', priority: 1, tail: true })
+        .returning('id')
+      const gid = Number(g.id)
+      const range = (start_id: number, end_id: number, extra: Record<string, unknown> = {}) => ({
+        group_id: gid,
+        start_id,
+        end_id,
+        next_id: start_id,
+        priority: TAIL,
+        ...extra,
+      })
+      await db.table('ranges').multiInsert([
+        // History, then a run of five slivers (the fifth is spooled: a break), two more, a capture range, the tail's rest.
+        range(1, 1000, { priority: 500, status: 'done' }),
+        range(1001, 1100),
+        range(1101, 1200),
+        range(1201, 1300),
+        range(1301, 1400),
+        range(1401, 1500, { spooled_chunks: 1 }),
+        range(1501, 1600),
+        range(1601, 1700),
+        range(1701, 1800, { capture: true }),
+        range(1801, 2600),
+        range(2601, 2700),
+        range(2701, 2800, { status: 'leased', node_id: 1 }),
+      ])
+      assert.deepEqual(await Planner.mergeTail(true), { candidates: 8, merged: 3, dropped: 5 })
+      assert.equal(
+        (await db.from('ranges').where('group_id', gid).count('* as n').first())!.n,
+        '12'
+      )
+      assert.deepEqual(await Planner.mergeTail(), { candidates: 8, merged: 3, dropped: 5 })
+      const ranges = await db
+        .from('ranges')
+        .where('group_id', gid)
+        .orderBy('start_id')
+        .select('start_id', 'end_id', 'next_id', 'status', 'priority', 'capture')
+      assert.deepEqual(
+        ranges.map((r) => [
+          Number(r.start_id),
+          Number(r.end_id),
+          Number(r.next_id),
+          r.status,
+          Number(r.priority),
+          r.capture,
+        ]),
+        [
+          [1, 1000, 1, 'done', 500, false],
+          [1001, 1400, 1001, 'planned', TAIL, false],
+          [1401, 1500, 1401, 'planned', TAIL, false],
+          [1501, 1700, 1501, 'planned', TAIL, false],
+          [1701, 1800, 1701, 'planned', TAIL, true],
+          [1801, 2700, 1801, 'planned', TAIL, false],
+          [2701, 2800, 2701, 'leased', TAIL, false],
+        ]
+      )
+      assert.deepEqual(await Planner.mergeTail(), { candidates: 3, merged: 0, dropped: 0 })
+    } finally {
+      coordinatorConfig.rangeArticles = saved
+    }
+  })
+})

tests/unit/coordinator/tail.spec.ts

@@ -1,5 +1,5 @@
 import { test } from '@japa/runner'
-import { tailRanges, wildmatOf } from '#services/planner'
+import { tailMerges, tailRanges, wildmatOf } from '#services/planner'
 
 const options = { rangeArticles: 1000, minArticles: 100, maxDelayMinutes: 15, priority: -1 }
 const now = new Date('2026-10-09T10:00:00Z')
@@ -7,7 +7,7 @@ const minutesAgo = (m: number) => new Date(now.getTime() - m * 60_000)
 
 test.group('coordinator / live tail', () => {
   test('a chunk of new articles, or a long enough wait, makes tail ranges', ({ assert }) => {
-    const rows = tailRanges(
+    const { rows, extended } = tailRanges(
       [
         // Enough new articles: planned at once, cut at rangeArticles.
         { id: 1, plannedTo: 5000, plannedAt: minutesAgo(2), high: 7500 },
@@ -32,6 +32,94 @@ test.group('coordinator / live tail', () => {
         [3, 101, 103, 101, -1],
       ]
     )
+    assert.deepEqual(extended, [])
+  })
+
+  test('an open last range grows up to rangeArticles before rows are added', ({ assert }) => {
+    const { rows, extended } = tailRanges(
+      [
+        // Room in the open range for everything new.
+        {
+          id: 1,
+          plannedTo: 5300,
+          plannedAt: minutesAgo(20),
+          high: 5350,
+          open: { id: 11, start: 5001 },
+        },
+        // The open range fills up, the rest is cut as rows.
+        {
+          id: 2,
+          plannedTo: 5300,
+          plannedAt: minutesAgo(2),
+          high: 7500,
+          open: { id: 12, start: 5001 },
+        },
+        // An open range already full: rows only.
+        {
+          id: 3,
+          plannedTo: 6000,
+          plannedAt: minutesAgo(2),
+          high: 6500,
+          open: { id: 13, start: 5001 },
+        },
+        // Not open (taken, spooled, another priority): rows.
+        { id: 4, plannedTo: 5300, plannedAt: minutesAgo(20), high: 5350, open: null },
+      ],
+      now,
+      options
+    )
+    assert.deepEqual(
+      extended.map((e) => [e.id, e.group_id, e.from, e.end_id]),
+      [
+        [11, 1, 5301, 5350],
+        [12, 2, 5301, 6000],
+      ]
+    )
+    assert.deepEqual(
+      rows.map((r) => [r.group_id, r.start_id, r.end_id]),
+      [
+        [2, 6001, 7000],
+        [2, 7001, 7500],
+        [3, 6001, 6500],
+        [4, 5301, 5350],
+      ]
+    )
+  })
+
+  test('adjacent ranges of a group and priority pack into rangeArticles; gaps and other priorities break a run', ({
+    assert,
+  }) => {
+    const r = (id: number, group_id: number, start_id: number, end_id: number, priority = -3) => ({
+      id,
+      group_id,
+      start_id,
+      end_id,
+      priority,
+    })
+    const merges = tailMerges(
+      [
+        // 1–300 + 301–600 + 601–900 fill 1000 with 901–1000; 1001–1100 starts the next run alone.
+        r(1, 1, 1, 300),
+        r(2, 1, 301, 600),
+        r(3, 1, 601, 900),
+        r(4, 1, 901, 1000),
+        r(5, 1, 1001, 1100),
+        // A gap, then two adjacent at another priority.
+        r(6, 1, 1200, 1250),
+        r(7, 1, 1251, 1300, -1),
+        r(8, 1, 1301, 1400, -1),
+        // Another group; one that would overflow stays alone.
+        r(9, 2, 1, 900),
+        r(10, 2, 901, 1100),
+        r(11, 2, 1101, 1150),
+      ],
+      1000
+    )
+    assert.deepEqual(merges, [
+      { id: 1, end_id: 1000, drop: [2, 3, 4] },
+      { id: 7, end_id: 1400, drop: [8] },
+      { id: 10, end_id: 1150, drop: [11] },
+    ])
   })
 
   test('LIST ACTIVE patterns by the first two name parts', ({ assert }) => {