From e779f67c773e40c57962ef3a86ddea6547bc9992 Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Sun, 25 Oct 2026 22:43:01 +0100 Subject: [PATCH] Prefetcher: corpus candidates a range of every group at a time The 512 candidates were the lowest ids at the lowest priority; a group's ranges have consecutive ids, so once a window's small groups were stored they were all one group's, two targets on two connections (10 ranges an hour). Ordered by the per-group rank now: the window spans 85 groups. Co-Authored-By: Claude Fable 5.1 --- app/services/prefetcher.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/services/prefetcher.ts b/app/services/prefetcher.ts index d2b0391..aa3a8e3 100644 --- a/app/services/prefetcher.ts +++ b/app/services/prefetcher.ts @@ -506,7 +506,10 @@ export class Prefetcher { * Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started * (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too); * as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which - * took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). A range found complete + * took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken + * a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's + * ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one + * group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete * gets stored_at and isn't read again. */ private async corpusTargets(rows: (q: any) => any, taken: Set): Promise { @@ -524,7 +527,7 @@ export class Prefetcher { .where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0)) .orWhere('r.status', 'done') ) - .orderBy([{ column: 'r.priority' }, { column: 'r.id' }]) + .orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id') .limit(ahead * 16) const targets: Target[] = [] for (const row of candidates) {