Files
usenet-indexer/tests/functional/planner_tail.spec.ts
T
Daan MeijerandClaude Fable 5.1 1810abd7d8
Test and deploy / test (push) Successful in 58s
Test and deploy / edge (push) Successful in 2m22s
Test and deploy / deploy (push) Successful in 2m53s
The tail planner grows a group's open range; ranges:merge-tail packs the slivers
A round added a row a group whenever 100k articles or 15 minutes had passed,
tuned for a tail leased within minutes; with nothing leasing the tail the rows
piled up (127k planned tail ranges of a median 14k articles, 97% adjacent). Now
a round grows the group's last range when it is a planned, unspooled, untaken
row at the tail's priority, up to RANGE_ARTICLES, and only then adds rows; a
range taken meanwhile gets the slice as a row. ranges:merge-tail (--dry-run)
packs adjacent planned, uncaptured, unspooled, untaken tail ranges of a group
and priority into ranges of RANGE_ARTICLES under the lease lock.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-27 10:46:47 +01:00

143 lines
4.8 KiB
TypeScript

import { test } from '@japa/runner'
import db from '@adonisjs/lucid/services/db'
import testUtils from '@adonisjs/core/services/test_utils'
import Planner from '#services/planner'
import coordinatorConfig from '#config/coordinator'
const TAIL = coordinatorConfig.tail.priority
test.group('Planner / tail ranges', (group) => {
group.each.setup(() => testUtils.db().withGlobalTransaction())
test('applyTail grows an open range, or adds the slice when the range was taken meanwhile', async ({
assert,
}) => {
const [g] = await db
.table('groups')
.insert({
name: 'alt.binaries.tail',
priority: 1,
tail: true,
planned_at: new Date('2026-10-27T08:00:00Z'),
})
.returning('id')
const [open] = await db
.table('ranges')
.insert({ group_id: g.id, start_id: 1, end_id: 300, next_id: 1, priority: TAIL })
.returning('id')
const [taken] = await db
.table('ranges')
.insert({
group_id: g.id,
start_id: 301,
end_id: 400,
next_id: 301,
priority: TAIL,
status: 'leased',
node_id: 1,
})
.returning('id')
const now = new Date('2026-10-27T09:00:00Z')
const planned = await Planner.applyTail(
{
rows: [
{ group_id: Number(g.id), start_id: 2001, end_id: 2500, next_id: 2001, priority: TAIL },
],
extended: [
{ id: Number(open.id), group_id: Number(g.id), from: 301, end_id: 2000 },
{ id: Number(taken.id), group_id: Number(g.id), from: 401, end_id: 600 },
],
},
now
)
assert.deepEqual(planned, { groups: 1, ranges: 2, extended: 1 })
const ranges = await db
.from('ranges')
.where('group_id', g.id)
.orderBy('start_id')
.select('start_id', 'end_id', 'status', 'priority')
assert.deepEqual(
ranges.map((r) => [Number(r.start_id), Number(r.end_id), r.status, Number(r.priority)]),
[
[1, 2000, 'planned', TAIL],
[301, 400, 'leased', TAIL],
[401, 600, 'planned', TAIL],
[2001, 2500, 'planned', TAIL],
]
)
const updated = await db.from('groups').where('id', g.id).firstOrFail()
assert.equal(Number(updated.high), 2500)
assert.equal(new Date(updated.planned_at).toISOString(), now.toISOString())
})
test('mergeTail packs adjacent planned tail ranges and leaves taken, spooled, capture and history ranges alone', async ({
assert,
}) => {
const saved = coordinatorConfig.rangeArticles
coordinatorConfig.rangeArticles = 1000
try {
const [g] = await db
.table('groups')
.insert({ name: 'alt.binaries.tail', priority: 1, tail: true })
.returning('id')
const gid = Number(g.id)
const range = (start_id: number, end_id: number, extra: Record<string, unknown> = {}) => ({
group_id: gid,
start_id,
end_id,
next_id: start_id,
priority: TAIL,
...extra,
})
await db.table('ranges').multiInsert([
// History, then a run of five slivers (the fifth is spooled: a break), two more, a capture range, the tail's rest.
range(1, 1000, { priority: 500, status: 'done' }),
range(1001, 1100),
range(1101, 1200),
range(1201, 1300),
range(1301, 1400),
range(1401, 1500, { spooled_chunks: 1 }),
range(1501, 1600),
range(1601, 1700),
range(1701, 1800, { capture: true }),
range(1801, 2600),
range(2601, 2700),
range(2701, 2800, { status: 'leased', node_id: 1 }),
])
assert.deepEqual(await Planner.mergeTail(true), { candidates: 8, merged: 3, dropped: 5 })
assert.equal(
(await db.from('ranges').where('group_id', gid).count('* as n').first())!.n,
'12'
)
assert.deepEqual(await Planner.mergeTail(), { candidates: 8, merged: 3, dropped: 5 })
const ranges = await db
.from('ranges')
.where('group_id', gid)
.orderBy('start_id')
.select('start_id', 'end_id', 'next_id', 'status', 'priority', 'capture')
assert.deepEqual(
ranges.map((r) => [
Number(r.start_id),
Number(r.end_id),
Number(r.next_id),
r.status,
Number(r.priority),
r.capture,
]),
[
[1, 1000, 1, 'done', 500, false],
[1001, 1400, 1001, 'planned', TAIL, false],
[1401, 1500, 1401, 'planned', TAIL, false],
[1501, 1700, 1501, 'planned', TAIL, false],
[1701, 1800, 1701, 'planned', TAIL, true],
[1801, 2700, 1801, 'planned', TAIL, false],
[2701, 2800, 2701, 'leased', TAIL, false],
]
)
assert.deepEqual(await Planner.mergeTail(), { candidates: 3, merged: 0, dropped: 0 })
} finally {
coordinatorConfig.rangeArticles = saved
}
})
})