A round added a row a group whenever 100k articles or 15 minutes had passed, tuned for a tail leased within minutes; with nothing leasing the tail the rows piled up (127k planned tail ranges of a median 14k articles, 97% adjacent). Now a round grows the group's last range when it is a planned, unspooled, untaken row at the tail's priority, up to RANGE_ARTICLES, and only then adds rows; a range taken meanwhile gets the slice as a row. ranges:merge-tail (--dry-run) packs adjacent planned, uncaptured, unspooled, untaken tail ranges of a group and priority into ranges of RANGE_ARTICLES under the lease lock. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
143 lines
4.8 KiB
TypeScript
143 lines
4.8 KiB
TypeScript
import { test } from '@japa/runner'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import testUtils from '@adonisjs/core/services/test_utils'
|
|
import Planner from '#services/planner'
|
|
import coordinatorConfig from '#config/coordinator'
|
|
|
|
const TAIL = coordinatorConfig.tail.priority
|
|
|
|
test.group('Planner / tail ranges', (group) => {
|
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
|
|
|
test('applyTail grows an open range, or adds the slice when the range was taken meanwhile', async ({
|
|
assert,
|
|
}) => {
|
|
const [g] = await db
|
|
.table('groups')
|
|
.insert({
|
|
name: 'alt.binaries.tail',
|
|
priority: 1,
|
|
tail: true,
|
|
planned_at: new Date('2026-10-27T08:00:00Z'),
|
|
})
|
|
.returning('id')
|
|
const [open] = await db
|
|
.table('ranges')
|
|
.insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, priority: TAIL })
|
|
.returning('id')
|
|
const [taken] = await db
|
|
.table('ranges')
|
|
.insert({
|
|
group_id: g.id,
|
|
start_id: 301,
|
|
end_id: 400,
|
|
next_id: 301,
|
|
priority: TAIL,
|
|
status: 'leased',
|
|
node_id: 1,
|
|
})
|
|
.returning('id')
|
|
const now = new Date('2026-10-27T09:00:00Z')
|
|
const planned = await Planner.applyTail(
|
|
{
|
|
rows: [
|
|
{ group_id: Number(g.id), start_id: 2001, end_id: 2500, next_id: 2001, priority: TAIL },
|
|
],
|
|
extended: [
|
|
{ id: Number(open.id), group_id: Number(g.id), from: 301, end_id: 2000 },
|
|
{ id: Number(taken.id), group_id: Number(g.id), from: 401, end_id: 600 },
|
|
],
|
|
},
|
|
now
|
|
)
|
|
assert.deepEqual(planned, { groups: 1, ranges: 2, extended: 1 })
|
|
const ranges = await db
|
|
.from('ranges')
|
|
.where('group_id', g.id)
|
|
.orderBy('start_id')
|
|
.select('start_id', 'end_id', 'status', 'priority')
|
|
assert.deepEqual(
|
|
ranges.map((r) => [Number(r.start_id), Number(r.end_id), r.status, Number(r.priority)]),
|
|
[
|
|
[1, 2000, 'planned', TAIL],
|
|
[301, 400, 'leased', TAIL],
|
|
[401, 600, 'planned', TAIL],
|
|
[2001, 2500, 'planned', TAIL],
|
|
]
|
|
)
|
|
const updated = await db.from('groups').where('id', g.id).firstOrFail()
|
|
assert.equal(Number(updated.high), 2500)
|
|
assert.equal(new Date(updated.planned_at).toISOString(), now.toISOString())
|
|
})
|
|
|
|
test('mergeTail packs adjacent planned tail ranges and leaves taken, spooled, capture and history ranges alone', async ({
|
|
assert,
|
|
}) => {
|
|
const saved = coordinatorConfig.rangeArticles
|
|
coordinatorConfig.rangeArticles = 1000
|
|
try {
|
|
const [g] = await db
|
|
.table('groups')
|
|
.insert({ name: 'alt.binaries.tail', priority: 1, tail: true })
|
|
.returning('id')
|
|
const gid = Number(g.id)
|
|
const range = (start_id: number, end_id: number, extra: Record<string, unknown> = {}) => ({
|
|
group_id: gid,
|
|
start_id,
|
|
end_id,
|
|
next_id: start_id,
|
|
priority: TAIL,
|
|
...extra,
|
|
})
|
|
await db.table('ranges').multiInsert([
|
|
// History, then a run of five slivers (the fifth is spooled: a break), two more, a capture range, the tail's rest.
|
|
range(1, 1000, { priority: 500, status: 'done' }),
|
|
range(1001, 1100),
|
|
range(1101, 1200),
|
|
range(1201, 1300),
|
|
range(1301, 1400),
|
|
range(1401, 1500, { spooled_chunks: 1 }),
|
|
range(1501, 1600),
|
|
range(1601, 1700),
|
|
range(1701, 1800, { capture: true }),
|
|
range(1801, 2600),
|
|
range(2601, 2700),
|
|
range(2701, 2800, { status: 'leased', node_id: 1 }),
|
|
])
|
|
assert.deepEqual(await Planner.mergeTail(true), { candidates: 8, merged: 3, dropped: 5 })
|
|
assert.equal(
|
|
(await db.from('ranges').where('group_id', gid).count('* as n').first())!.n,
|
|
'12'
|
|
)
|
|
assert.deepEqual(await Planner.mergeTail(), { candidates: 8, merged: 3, dropped: 5 })
|
|
const ranges = await db
|
|
.from('ranges')
|
|
.where('group_id', gid)
|
|
.orderBy('start_id')
|
|
.select('start_id', 'end_id', 'next_id', 'status', 'priority', 'capture')
|
|
assert.deepEqual(
|
|
ranges.map((r) => [
|
|
Number(r.start_id),
|
|
Number(r.end_id),
|
|
Number(r.next_id),
|
|
r.status,
|
|
Number(r.priority),
|
|
r.capture,
|
|
]),
|
|
[
|
|
[1, 1000, 1, 'done', 500, false],
|
|
[1001, 1400, 1001, 'planned', TAIL, false],
|
|
[1401, 1500, 1401, 'planned', TAIL, false],
|
|
[1501, 1700, 1501, 'planned', TAIL, false],
|
|
[1701, 1800, 1701, 'planned', TAIL, true],
|
|
[1801, 2700, 1801, 'planned', TAIL, false],
|
|
[2701, 2800, 2701, 'leased', TAIL, false],
|
|
]
|
|
)
|
|
assert.deepEqual(await Planner.mergeTail(), { candidates: 3, merged: 0, dropped: 0 })
|
|
} finally {
|
|
coordinatorConfig.rangeArticles = saved
|
|
}
|
|
})
|
|
})
|