Merge branch 'rebuild' into voyager-3
Test and deploy / test (push) Successful in 49s
Test and deploy / edge (push) Successful in 1m51s
Test and deploy / deploy (push) Skipped

This commit is contained in:
Daan Meijer committed 2026-10-25 22:43:14 +01:00
commit cf51db047f
1 file changed
+5 -2
+5 -2
View File
@@ -506,7 +506,10 @@ export class Prefetcher {
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). A range found complete
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
* gets stored_at and isn't read again.
*/
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
@@ -524,7 +527,7 @@ export class Prefetcher {
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
.orWhere('r.status', 'done')
)
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
.limit(ahead * 16)
const targets: Target[] = []
for (const row of candidates) {