Daan MeijerandClaude Opus 5.5
d5f168d8d2
- groups:add / ranges:plan / nodes:create / coordinator:run commands - the node API: lease, the range's chunks as one framed stream from the spool, batches, heartbeat, finish, release (node guard, token per node) - edge:node: needs COORDINATOR_URL and NODE_TOKEN only (placeholders for the rest), posts a batch every 10 chunks, gives its range back on SIGTERM - releases are gathered per batch: 1.04 chunk rows per release at 10 chunks per batch (1.35 when built per XOVER chunk) - partitions are checked per batch instead of remembered Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
65 lines
2.6 KiB
TypeScript
65 lines
2.6 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import coordinatorConfig from '#config/coordinator'
|
|
import NntpPool from '#services/nntp_pool'
|
|
|
|
/**
|
|
* The indexed groups and their ranges (REBUILD.md, group list, coordinator).
|
|
* New groups get priorities after the existing ones (owner rule). A group's
|
|
* span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is
|
|
* its position in its group's history (0 oldest … 1000 newest), so all groups
|
|
* start with their oldest era. Planning again only adds ranges for new articles.
|
|
*/
|
|
class Planner {
|
|
async addGroups(names: string[]): Promise<number> {
|
|
let added = 0
|
|
for (const name of names) {
|
|
const { rows } = await db.rawQuery(
|
|
`INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`,
|
|
[name]
|
|
)
|
|
added += rows.length
|
|
}
|
|
return added
|
|
}
|
|
|
|
/** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */
|
|
async largestGroups(pattern: string, top: number): Promise<string[]> {
|
|
const groups = await NntpPool.use((client) => client.listActive(pattern))
|
|
return groups
|
|
.filter((g) => g.high > g.low)
|
|
.sort((a, b) => b.high - b.low - (a.high - a.low))
|
|
.slice(0, top)
|
|
.map((g) => g.name)
|
|
}
|
|
|
|
async planGroup(group: { id: number; name: string }): Promise<number> {
|
|
const info = await NntpPool.use((client) => client.group(group.name))
|
|
const size = coordinatorConfig.rangeArticles
|
|
const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
|
|
const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
|
|
const span = Math.max(1, info.high - info.low)
|
|
const rows = []
|
|
for (let start = from; start <= info.high; start += size) {
|
|
rows.push({
|
|
group_id: group.id,
|
|
start_id: start,
|
|
end_id: Math.min(start + size - 1, info.high),
|
|
next_id: start,
|
|
priority: Math.floor((1000 * (start - info.low)) / span),
|
|
})
|
|
}
|
|
for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
|
|
await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
|
|
return rows.length
|
|
}
|
|
|
|
async planAll(): Promise<{ groups: number; ranges: number }> {
|
|
const groups = await db.from('groups').where('active', true).orderBy('priority').select('id', 'name')
|
|
let ranges = 0
|
|
for (const group of groups) ranges += await this.planGroup({ id: Number(group.id), name: group.name })
|
|
return { groups: groups.length, ranges }
|
|
}
|
|
}
|
|
|
|
export default new Planner()
|