daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

The tail planner grows a group's open range; ranges:merge-tail packs the slivers
Test and deploy / test (push) Successful in 58s
Details
Test and deploy / edge (push) Successful in 2m22s
Details
Test and deploy / deploy (push) Successful in 2m53s
Details

Browse Source 
A round added a row a group whenever 100k articles or 15 minutes had passed,
tuned for a tail leased within minutes; with nothing leasing the tail the rows
piled up (127k planned tail ranges of a median 14k articles, 97% adjacent). Now
a round grows the group's last range when it is a planned, unspooled, untaken
row at the tail's priority, up to RANGE_ARTICLES, and only then adds rows; a
range taken meanwhile gets the slice as a row. ranges:merge-tail (--dry-run)
packs adjacent planned, uncaptured, unspooled, untaken tail ranges of a group
and priority into ranges of RANGE_ARTICLES under the lease lock.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-27 10:46:47 +01:00
1 parent e3957e5af5
commit 1810abd7d8
7 files changed
+423 -29
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/planner.ts
+150 -22
View File
@@ -75,32 +75,99 @@ class Planner {
* numbers come from LIST ACTIVE, one command per name prefix (GROUP per
* group took minutes for 1000 groups).
*/
async planTail(now = new Date()): Promise<{ groups: number; ranges: number }> {
async planTail(now = new Date()): Promise<TailPlanned> {
const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
if (groups.length === 0) return { groups: 0, ranges: 0 }
const { rows: ends } = await db.rawQuery(`SELECT DISTINCT ON (group_id) group_id, end_id FROM ranges ORDER BY group_id, start_id DESC`)
const plannedTo = new Map<number, number>(ends.map((r: any) => [Number(r.group_id), Number(r.end_id)]))
if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
const { rows: ends } = await db.rawQuery(
`SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
(status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
FROM ranges ORDER BY group_id, start_id DESC`,
[coordinatorConfig.tail.priority]
)
const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
)
const highs = new Map<string, number>()
await NntpPool.use(async (client) => {
for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
}
})
const rows = tailRanges(
groups.map((g) => ({ id: Number(g.id), plannedTo: plannedTo.get(Number(g.id)) ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null })),
const plan = tailRanges(
groups.map((g) => {
const l = last.get(Number(g.id))
return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
}),
now,
{ rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
)
if (rows.length === 0) return { groups: 0, ranges: 0 }
await db.transaction(async (trx) => {
await trx.table('ranges').multiInsert(rows)
for (const id of new Set(rows.map((r) => r.group_id))) {
const high = Math.max(...rows.filter((r) => r.group_id === id).map((r) => r.end_id))
await trx.from('groups').where('id', id).update({ high, planned_at: now })
}
})
return { groups: new Set(rows.map((r) => r.group_id)).size, ranges: rows.length }
return this.applyTail(plan, now)
}
/** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
const rows = [...plan.rows]
let extended = 0
if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
await db.transaction(async (trx) => {
for (const e of plan.extended) {
const { rows: done } = await trx.rawQuery(
`UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
[e.end_id, e.id, e.from - 1]
)
if (done.length > 0) extended++
else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
}
if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
const highs = new Map<number, number>()
for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
})
return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
}
/**
* Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
* RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
* (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
*/
async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
const { rows: candidates } = await db.rawQuery(
`SELECT id, group_id, start_id, end_id, priority FROM ranges
WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
ORDER BY group_id, start_id`
)
const merges = tailMerges(
candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
coordinatorConfig.rangeArticles
)
const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
await db.transaction(async (trx) => {
// The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
const { rows: gone } = await trx.rawQuery(
`DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
[merges.flatMap((m) => m.drop)]
)
if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
const { rows: grown } = await trx.rawQuery(
`UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
[merges.map((m) => m.id), merges.map((m) => m.end_id)]
)
if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
})
return { candidates: candidates.length, merged: merges.length, dropped }
}
}
export interface TailPlanned {
groups: number
/** Rows added. */
ranges: number
/** Open ranges grown. */
extended: number
}
/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
@@ -117,29 +184,90 @@ export interface TailGroup {
plannedAt: Date
/** The server's newest article (null: not listed). */
high: number | null
/**
* Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
* nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
*/
open?: { id: number; start: number } | null
}
export interface TailRow {
group_id: number
start_id: number
end_id: number
next_id: number
priority: number
}
export interface TailPlan {
/** New ranges. */
rows: TailRow[]
/** Open ranges grown to end_id (from: the first new article, the old end + 1). */
extended: { id: number; group_id: number; from: number; end_id: number }[]
}
/**
* The tail's new ranges: per group, the articles after its last range, in
* ranges of at most `rangeArticles`, once there are `minArticles` of them or
* the group's last planning is `maxDelayMinutes` ago.
* The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
* the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
* tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
*/
export function tailRanges(
groups: TailGroup[],
now: Date,
options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
) {
const rows: { group_id: number; start_id: number; end_id: number; next_id: number; priority: number }[] = []
): TailPlan {
const rows: TailRow[] = []
const extended: TailPlan['extended'] = []
for (const g of groups) {
if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
const waiting = g.high - g.plannedTo
const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
for (let start = g.plannedTo + 1; start <= g.high; start += options.rangeArticles) {
let start = g.plannedTo + 1
if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
start = end + 1
}
for (; start <= g.high; start += options.rangeArticles) {
rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
}
}
return rows
return { rows, extended }
}
export interface TailMerge {
/** The run's first range, grown to end_id. */
id: number
end_id: number
/** The ranges it absorbed. */
drop: number[]
}
/**
* Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
* `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
*/
export function tailMerges(
ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
rangeArticles: number
): TailMerge[] {
const merges: TailMerge[] = []
let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
const flush = () => {
if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
}
for (const r of ranges) {
if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
cur.end_id = r.end_id
cur.drop.push(r.id)
continue
}
flush()
cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
}
flush()
return merges
}
export default new Planner()