From 2b4f673ddbfd749ac760915dda0afcb27bb745c5 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 02:59:30 +0200 Subject: [PATCH] Rebuild: index lookups for the merger's neighbours and the prefetcher's ranges The merger's neighbour query matched "end_id = ? OR start_id = ?", which no index serves: it read every range of the group per file, 5.5 s a round and most of Postgres's time in an end-to-end run. Now the range holding the article before and after, by (group_id, start_id): 0.08 ms. The prefetcher's leased and started ranges (every 2 s) get partial indexes: 15-25 ms -> <0.1 ms with 145k planned ranges. Co-Authored-By: Claude Opus 5.5 --- app/services/fragment_merger.ts | 15 ++++++++++----- .../1770000001100_ranges_prefetch_indexes.ts | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 database/migrations/1770000001100_ranges_prefetch_indexes.ts diff --git a/app/services/fragment_merger.ts b/app/services/fragment_merger.ts index 6ffc18d..e4596c9 100644 --- a/app/services/fragment_merger.ts +++ b/app/services/fragment_merger.ts @@ -120,17 +120,22 @@ class FragmentMerger { }) const spans = lists.map(span) const groups = [...new Set(spans.map((s) => s.group))] + // The range holding the article before and the one after, each by the (group_id, start_id) index: + // "end_id = ? OR start_id = ?" read every range of the group per file (5.5 s a round; LOG.md 2026-10-09). const { rows } = await db.rawQuery( - `SELECT r.group_id, r.start_id, r.end_id, r.status FROM ranges r - JOIN unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start) - ON r.group_id = s.group_id AND (r.end_id = s.before_end OR r.start_id = s.after_start)`, + `SELECT s.group_id, s.before_end, s.after_start, b.status AS before, a.status AS after + FROM unnest(?::int[], ?::bigint[], ?::bigint[]) AS s(group_id, before_end, after_start) + LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.before_end AND r.end_id >= s.before_end + ORDER BY r.start_id DESC LIMIT 1) b ON true + LEFT JOIN LATERAL (SELECT r.status FROM ranges r WHERE r.group_id = s.group_id AND r.start_id <= s.after_start AND r.end_id >= s.after_start + ORDER BY r.start_id DESC LIMIT 1) a ON true`, [spans.map((s) => s.group), spans.map((s) => s.first - 1), spans.map((s) => s.last + 1)] ) const byEnd = new Map() const byStart = new Map() for (const r of rows) { - byEnd.set(`${r.group_id}:${r.end_id}`, r.status) - byStart.set(`${r.group_id}:${r.start_id}`, r.status) + if (r.before !== null) byEnd.set(`${r.group_id}:${r.before_end}`, r.before) + if (r.after !== null) byStart.set(`${r.group_id}:${r.after_start}`, r.after) } const lows = new Map((await db.from('groups').whereIn('id', groups).select('id', 'low')).map((g) => [Number(g.id), g.low === null ? null : Number(g.low)])) return (list: (typeof lists)[number]) => { diff --git a/database/migrations/1770000001100_ranges_prefetch_indexes.ts b/database/migrations/1770000001100_ranges_prefetch_indexes.ts new file mode 100644 index 0000000..5952dad --- /dev/null +++ b/database/migrations/1770000001100_ranges_prefetch_indexes.ts @@ -0,0 +1,18 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * The prefetcher's ranges every 2 s (REBUILD.md, prefetch): the leased ones + * and the planned ones it started spooling. Without these it read all + * ranges each time (~15–25 ms with 145k planned; LOG.md 2026-10-09). + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_leased_idx ON ranges (leased_at) WHERE status = 'leased'`) + this.schema.raw(`CREATE INDEX IF NOT EXISTS ranges_spooling_idx ON ranges (priority, id) WHERE status = 'planned' AND spooled_chunks > 0`) + } + + async down() { + this.schema.raw(`DROP INDEX IF EXISTS ranges_spooling_idx`) + this.schema.raw(`DROP INDEX IF EXISTS ranges_leased_idx`) + } +}