Daan MeijerandClaude Opus 5.5
31493a98be
The neighbour check was three queries per file (~2,000 files/minute); now one per round of 20k files, with a cursor so waiting files don't take every round's places, and rounds until nothing is left or 30 s. On the dev backlog (old-style fragments): ~18k files/minute in the 30 s budget; at full throughput the new fragment rule leaves ~1,200 files/minute. A group that fails GROUP (gone from the server) no longer stops planning the others; edge:bench's groups are inactive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
73 lines
2.9 KiB
TypeScript
73 lines
2.9 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
|
import logger from '@adonisjs/core/services/logger'
|
|
import coordinatorConfig from '#config/coordinator'
|
|
import NntpPool from '#services/nntp_pool'
|
|
|
|
/**
|
|
* The indexed groups and their ranges (REBUILD.md, group list, coordinator).
|
|
* New groups get priorities after the existing ones (owner rule). A group's
|
|
* span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is
|
|
* its position in its group's history (0 oldest … 1000 newest), so all groups
|
|
* start with their oldest era. Planning again only adds ranges for new articles.
|
|
*/
|
|
class Planner {
|
|
async addGroups(names: string[]): Promise<number> {
|
|
let added = 0
|
|
for (const name of names) {
|
|
const { rows } = await db.rawQuery(
|
|
`INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`,
|
|
[name]
|
|
)
|
|
added += rows.length
|
|
}
|
|
return added
|
|
}
|
|
|
|
/** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */
|
|
async largestGroups(pattern: string, top: number): Promise<string[]> {
|
|
const groups = await NntpPool.use((client) => client.listActive(pattern))
|
|
return groups
|
|
.filter((g) => g.high > g.low)
|
|
.sort((a, b) => b.high - b.low - (a.high - a.low))
|
|
.slice(0, top)
|
|
.map((g) => g.name)
|
|
}
|
|
|
|
async planGroup(group: { id: number; name: string }): Promise<number> {
|
|
const info = await NntpPool.use((client) => client.group(group.name))
|
|
const size = coordinatorConfig.rangeArticles
|
|
const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
|
|
const from = Math.max(info.low, Number(last?.end ?? 0) + 1)
|
|
const span = Math.max(1, info.high - info.low)
|
|
const rows = []
|
|
for (let start = from; start <= info.high; start += size) {
|
|
rows.push({
|
|
group_id: group.id,
|
|
start_id: start,
|
|
end_id: Math.min(start + size - 1, info.high),
|
|
next_id: start,
|
|
priority: Math.floor((1000 * (start - info.low)) / span),
|
|
})
|
|
}
|
|
for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
|
|
await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
|
|
return rows.length
|
|
}
|
|
|
|
async planAll(): Promise<{ groups: number; ranges: number }> {
|
|
const groups = await db.from('groups').where('active', true).orderBy('priority').select('id', 'name')
|
|
let ranges = 0
|
|
for (const group of groups) {
|
|
// One group failing (e.g. gone from the server) doesn't stop the others.
|
|
try {
|
|
ranges += await this.planGroup({ id: Number(group.id), name: group.name })
|
|
} catch (error: any) {
|
|
logger.warn(`Planning ${group.name}: ${error.message}`)
|
|
}
|
|
}
|
|
return { groups: groups.length, ranges }
|
|
}
|
|
}
|
|
|
|
export default new Planner()
|