Daan MeijerandClaude Fable 5.1
4af5e06400
The lease bounds alone don't select a window: the history planner's oldest buckets are priority 0-2 as well, so the window worker at 0..0 also took 2026 history at 0 (25k ranges in 40 groups, fetched live) while 1,739 ranges of the 2016 corpus window sat at voyager 2's leftover priorities (done when the window was marked, planned again by the reset). With LEASE_CAPTURE_ONLY the lease, next and the prefetcher's run-ahead take capture ranges only; the run-ahead now also keeps to the lease bounds. Test: a group with history at 0 only isn't leased, the history range after the window isn't handed out. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
594 lines
24 KiB
TypeScript
594 lines
24 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import NntpPool from '#services/nntp_pool'
|
||
import Spool, { type ChunkSpan } from '#services/spool'
|
||
import Corpus, { type CorpusFilter } from '#services/corpus'
|
||
import RuleSets from '#services/rule_sets'
|
||
import CatalogSpace from '#services/catalog_space'
|
||
import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
|
||
|
||
interface Target {
|
||
id: number
|
||
group: string
|
||
/** Chunk starts to fetch without compression (ranges.plain_chunks). */
|
||
plain: Set<number>
|
||
/** Chunks not spooled yet, in order. */
|
||
missing: ChunkSpan[]
|
||
/** All of the range's chunks, from its start. */
|
||
total: number
|
||
start: number
|
||
end: number
|
||
/** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
|
||
capture: boolean
|
||
/** Fetched into the corpus only, not the spool: a capture range no worker leases soon (corpus.ahead). */
|
||
corpusOnly?: boolean
|
||
}
|
||
|
||
/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
|
||
const STALL_SAMPLE_MS = 15_000
|
||
const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS
|
||
|
||
/** Connection ages the report counts slow first requests by, in minutes. */
|
||
const AGE_BUCKETS = [2, 10, Infinity]
|
||
|
||
/** How often the targets are read again from the database. */
|
||
const REFRESH_MS = 2000
|
||
/** How often the spool's size is measured (a stat per file). */
|
||
const SIZE_MS = 30_000
|
||
|
||
/**
|
||
* Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
|
||
* Every connection takes the next missing chunk: the leased ranges' first
|
||
* (a worker waits for them), then the next planned ranges, with at most
|
||
* PREFETCH_PER_RANGE connections on a range and PREFETCH_PER_GROUP on a
|
||
* group (Eweka answers faster spread out), about SPOOL_AHEAD_PER_LEASE per
|
||
* active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
|
||
* connection fills ~26 k overview lines/s, so a range on one connection
|
||
* kept its worker waiting (LOG.md 2026-10-09). A chunk that takes long is
|
||
* asked again on a spare connection (fetchHedged).
|
||
*/
|
||
export class Prefetcher {
|
||
/**
|
||
* This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
|
||
* process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
|
||
* LOG.md 2026-10-09), less than 50 connections bring.
|
||
*/
|
||
constructor(private shard = { index: 0, count: 1 }) {}
|
||
|
||
private targets: Target[] = []
|
||
private claimed = new Set<string>()
|
||
/** Chunks being fetched per range and per group (spool.perRange, spool.perGroup). */
|
||
private perRange = new Map<number, number>()
|
||
private perGroup = new Map<string, number>()
|
||
private running = 0
|
||
private refreshedAt = 0
|
||
private stopped = false
|
||
private spoolBytes = 0
|
||
private sizedAt = 0
|
||
/** Fetched since the last report: chunks, articles, ms spent fetching. */
|
||
private stats = this.freshStats()
|
||
/** Loops in a fetch or a spool write right now. */
|
||
private fetchingNow = 0
|
||
private writingNow = 0
|
||
private sampler?: NodeJS.Timeout
|
||
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
|
||
private inFlight = new Set<{
|
||
rangeId: number
|
||
from: number
|
||
to: number
|
||
since: number
|
||
client: XoverClient
|
||
}>()
|
||
/** Capture ranges found complete in the corpus (not looked at again). */
|
||
private stored = new Set<number>()
|
||
|
||
private freshStats() {
|
||
return {
|
||
chunks: 0,
|
||
/** Chunks read from the corpus instead of fetched. */
|
||
stored: 0,
|
||
/** Chunks fetched into the corpus only (corpus.ahead). */
|
||
captured: 0,
|
||
articles: 0,
|
||
fetchMs: 0,
|
||
hedged: 0,
|
||
hedgeWon: 0,
|
||
/** First requests per connection-age bucket, and how many of them were slow. */
|
||
byAge: AGE_BUCKETS.map(() => ({ requests: 0, slow: 0 })),
|
||
/** Spool writes and the range's update, ms. */
|
||
writeMs: 0,
|
||
/** Per answered request, ms: GROUP, XOVER until its status line, the transfer; transfers with a silence over 5 s. */
|
||
requests: 0,
|
||
groupMs: 0,
|
||
replyMs: 0,
|
||
bodyMs: 0,
|
||
silent: 0,
|
||
/** Once a second: loops fetching, writing, not running; pool connections with a command in flight. */
|
||
samples: 0,
|
||
fetching: 0,
|
||
writing: 0,
|
||
idle: 0,
|
||
inFlight: 0,
|
||
inFlightMin: Infinity,
|
||
/** Requests given up as stalled (nntpStallBytesPerMinute). */
|
||
deadlines: 0,
|
||
ranges: 0,
|
||
groups: 0,
|
||
since: Date.now(),
|
||
}
|
||
}
|
||
|
||
/** The prefetcher's loops: the pool less the connections kept for hedging. */
|
||
private get loops() {
|
||
return Math.max(1, NntpPool.size - coordinatorConfig.nntpHedgeConnections)
|
||
}
|
||
|
||
/** One round: refreshes the targets now and then, and puts idle connections to work. */
|
||
async tick() {
|
||
this.sampler ??= setInterval(() => this.sample(), 1000).unref()
|
||
if (Date.now() - this.refreshedAt >= REFRESH_MS) {
|
||
this.refreshedAt = Date.now()
|
||
this.targets = await this.readTargets()
|
||
}
|
||
this.report()
|
||
while (!this.stopped && this.running < this.loops && this.peek()) {
|
||
this.running++
|
||
this.work()
|
||
.catch((error) => logger.warn(`Prefetching: ${error.message}`))
|
||
.finally(() => this.running--)
|
||
}
|
||
// Work left with every loop running: the pool's share could be bigger.
|
||
if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
|
||
}
|
||
|
||
stop() {
|
||
this.stopped = true
|
||
clearInterval(this.sampler)
|
||
}
|
||
|
||
get active() {
|
||
return this.running
|
||
}
|
||
|
||
/** One connection's loop: chunks until there is nothing left to fetch. */
|
||
private async work() {
|
||
while (!this.stopped) {
|
||
// The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
|
||
// Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
|
||
if (this.running > this.loops) return
|
||
const next = this.take()
|
||
if (!next) return
|
||
const { target, span } = next
|
||
try {
|
||
const fetchedAt = Date.now()
|
||
this.fetchingNow++
|
||
const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
|
||
await client.group(target.group)
|
||
return client.xoverRaw(span.from, span.to)
|
||
}
|
||
const corpusRange = { group: target.group, start: target.start, end: target.end }
|
||
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
|
||
let data =
|
||
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||
const stored = data !== null
|
||
try {
|
||
if (!data)
|
||
data = target.plain.has(span.from)
|
||
? await NntpPool.usePlain(fetch)
|
||
: await this.fetchHedged(target.id, target.group, span)
|
||
} finally {
|
||
this.fetchingNow--
|
||
}
|
||
if (stored) {
|
||
this.stats.stored++
|
||
} else {
|
||
this.stats.chunks++
|
||
this.stats.articles += span.to - span.from + 1
|
||
this.stats.fetchMs += Date.now() - fetchedAt
|
||
if (target.capture)
|
||
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
|
||
}
|
||
if (target.corpusOnly) {
|
||
this.stats.captured++
|
||
continue
|
||
}
|
||
const writtenAt = Date.now()
|
||
this.writingNow++
|
||
try {
|
||
const existed = await Spool.has(target.id, span)
|
||
await Spool.write(target.id, span, data)
|
||
if (existed) continue
|
||
this.spoolBytes += data.length
|
||
const { rows } = await db.rawQuery(
|
||
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
|
||
[target.id]
|
||
)
|
||
if (Number(rows[0]?.spooled_chunks) >= target.total)
|
||
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||
} finally {
|
||
this.writingNow--
|
||
this.stats.writeMs += Date.now() - writtenAt
|
||
}
|
||
} finally {
|
||
this.claimed.delete(`${target.id}:${span.from}`)
|
||
this.count(target, -1)
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* A chunk over a pooled connection. Not in after nntpHedgeAfterMs, it's
|
||
* asked again on a free connection (if there is one): Eweka stalls single
|
||
* requests, and the same chunk on another connection mostly comes in
|
||
* seconds (LOG.md 2026-10-09 (15)). The first answer wins; the other
|
||
* request can't be cancelled, so its connection is closed (the pool opens
|
||
* a fresh one).
|
||
*/
|
||
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
|
||
let age: number | undefined
|
||
const request = (
|
||
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
|
||
first: boolean
|
||
): Request<Buffer> | null => {
|
||
let client: XoverClient | undefined
|
||
let cancelled = false
|
||
const done = start(async (c) => {
|
||
// Given up before it got a connection: the connection goes back unused.
|
||
if (cancelled) return Buffer.alloc(0)
|
||
client = c
|
||
if (first) age = NntpPool.ageOf(c)
|
||
const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
|
||
this.inFlight.add(entry)
|
||
// Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall.
|
||
let late = false
|
||
const samples: number[] = []
|
||
const watch = setInterval(() => {
|
||
samples.push(c.buffered)
|
||
if (samples.length <= STALL_SAMPLES) return
|
||
const minuteAgo = samples.shift()!
|
||
if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) {
|
||
late = true
|
||
this.stats.deadlines++
|
||
c.close()
|
||
}
|
||
}, STALL_SAMPLE_MS)
|
||
let data: Buffer
|
||
let groupMs = 0
|
||
try {
|
||
const groupAt = performance.now()
|
||
await c.group(group)
|
||
groupMs = performance.now() - groupAt
|
||
data = await c.xoverRaw(span.from, span.to)
|
||
} catch (error) {
|
||
if (!late) throw error
|
||
throw new Error(
|
||
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
|
||
)
|
||
} finally {
|
||
clearInterval(watch)
|
||
this.inFlight.delete(entry)
|
||
}
|
||
if (!cancelled) {
|
||
const t = c.lastTiming
|
||
this.stats.requests++
|
||
this.stats.groupMs += groupMs
|
||
this.stats.replyMs += t.replyMs
|
||
this.stats.bodyMs += t.bodyMs
|
||
if (t.longestGapMs > 5000) this.stats.silent++
|
||
}
|
||
return data
|
||
})
|
||
if (!done) return null
|
||
done.catch(() => {})
|
||
return {
|
||
done,
|
||
// The request can't be called off on the wire: its connection is closed, and the pool opens a fresh one.
|
||
cancel: () => {
|
||
cancelled = true
|
||
client?.close()
|
||
},
|
||
}
|
||
}
|
||
const result = await hedge(
|
||
() => request((work) => NntpPool.use(work), true)!,
|
||
() => request((work) => NntpPool.tryUse(work), false),
|
||
coordinatorConfig.nntpHedgeAfterMs
|
||
)
|
||
if (age !== undefined) {
|
||
const bucket = this.stats.byAge[AGE_BUCKETS.findIndex((minutes) => age! < minutes * 60_000)]
|
||
bucket.requests++
|
||
if (result.late) bucket.slow++
|
||
}
|
||
if (result.hedged) this.stats.hedged++
|
||
if (result.secondWon) this.stats.hedgeWon++
|
||
return result.value
|
||
}
|
||
|
||
/** The longest-running request: range, chunk, how long, how much came in. */
|
||
private oldest() {
|
||
let oldest:
|
||
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
|
||
if (!oldest) return 'none in flight'
|
||
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
|
||
}
|
||
|
||
private sample() {
|
||
const s = this.stats
|
||
s.samples++
|
||
s.fetching += this.fetchingNow
|
||
s.writing += this.writingNow
|
||
s.idle += Math.max(0, this.loops - this.fetchingNow - this.writingNow)
|
||
s.inFlight += NntpPool.busyCount
|
||
s.inFlightMin = Math.min(s.inFlightMin, NntpPool.busyCount)
|
||
s.ranges += this.perRange.size
|
||
s.groups += this.perGroup.size
|
||
}
|
||
|
||
/** A line a minute: what the connections fetched, and how long a chunk took. */
|
||
private report() {
|
||
const seconds = (Date.now() - this.stats.since) / 1000
|
||
if (seconds < 60) return
|
||
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
|
||
if (chunks > 0) {
|
||
const ages = byAge
|
||
.map((b, i) =>
|
||
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
|
||
'– min',
|
||
'+ min'
|
||
)
|
||
)
|
||
.join(', ')
|
||
logger.info(
|
||
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
|
||
`${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
|
||
`hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
|
||
(this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus` : '') +
|
||
(this.stats.captured > 0 ? `; ${this.stats.captured} fetched into the corpus only` : '') +
|
||
'.'
|
||
)
|
||
const st = this.stats
|
||
const avg = (n: number) => (samples > 0 ? n / samples : 0)
|
||
const per = (ms: number) => (requests > 0 ? (ms / requests / 1000).toFixed(1) : '-')
|
||
logger.info(
|
||
`Connections: ${avg(st.inFlight).toFixed(1)} of ${NntpPool.size} with a command in flight (min ${st.inFlightMin}), ` +
|
||
`on ${avg(st.ranges).toFixed(1)} ranges of ${avg(st.groups).toFixed(1)} groups; ` +
|
||
`loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` +
|
||
`a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` +
|
||
`${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` +
|
||
`${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.`
|
||
)
|
||
}
|
||
this.stats = this.freshStats()
|
||
}
|
||
|
||
private peek(): boolean {
|
||
return this.targets.some(
|
||
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
|
||
)
|
||
}
|
||
|
||
/** Whether the target is under its range's and its group's limit of connections. */
|
||
private open(target: Target) {
|
||
const { perRange, perGroup } = coordinatorConfig.spool
|
||
return (
|
||
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
|
||
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||
)
|
||
}
|
||
|
||
private count(target: Target, by: number) {
|
||
const range = (this.perRange.get(target.id) ?? 0) + by
|
||
const group = (this.perGroup.get(target.group) ?? 0) + by
|
||
if (range > 0) this.perRange.set(target.id, range)
|
||
else this.perRange.delete(target.id)
|
||
if (group > 0) this.perGroup.set(target.group, group)
|
||
else this.perGroup.delete(target.group)
|
||
}
|
||
|
||
/** The first unclaimed missing chunk, in target order, of a target under its limits. */
|
||
private take(): { target: Target; span: ChunkSpan } | null {
|
||
for (const target of this.targets) {
|
||
if (!this.open(target)) continue
|
||
const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
|
||
if (index === -1) continue
|
||
const [span] = target.missing.splice(index, 1)
|
||
this.claimed.add(`${target.id}:${span.from}`)
|
||
this.count(target, 1)
|
||
return { target, span }
|
||
}
|
||
return null
|
||
}
|
||
|
||
private async readTargets(): Promise<Target[]> {
|
||
if (Date.now() - this.sizedAt >= SIZE_MS) {
|
||
this.sizedAt = Date.now()
|
||
this.spoolBytes = await Spool.size()
|
||
}
|
||
const mine = (q: any) =>
|
||
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
|
||
const rows = (q: any) =>
|
||
mine(
|
||
q
|
||
.from('ranges as r')
|
||
.join('groups as g', 'g.id', 'r.group_id')
|
||
.select(
|
||
'r.id',
|
||
'g.name as group',
|
||
'r.start_id',
|
||
'r.end_id',
|
||
'r.next_id',
|
||
'r.plain_chunks',
|
||
'r.capture'
|
||
)
|
||
)
|
||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||
// The groups workers hold (group_leases; REBUILD.md design 1): the prefetcher runs ahead of their cursors.
|
||
const held: number[] = (
|
||
await db
|
||
.from('group_leases')
|
||
.whereNotNull('node_id')
|
||
.where(
|
||
'heartbeat_at',
|
||
'>=',
|
||
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
|
||
)
|
||
.select('group_id')
|
||
).map((l) => Number(l.group_id))
|
||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||
const ahead = Math.max(
|
||
coordinatorConfig.spool.minAhead,
|
||
coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
|
||
)
|
||
const started = await rows(db.query())
|
||
.where('r.status', 'planned')
|
||
.where('r.spooled_chunks', '>', 0)
|
||
.whereNull('r.spooled_at')
|
||
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||
const [{ ready }] = await mine(
|
||
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
|
||
).count('* as ready')
|
||
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
|
||
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
|
||
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
|
||
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
|
||
const { perRange } = coordinatorConfig.spool
|
||
const short =
|
||
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||
const want = Math.max(ahead - Number(ready), short)
|
||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||
const full = await CatalogSpace.full()
|
||
// Ahead of each held group's cursor: its next ranges in article order (within what the group leases hand out:
|
||
// LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY), a range of every held group before any group's second; without
|
||
// held groups (no window worker running) nothing is fetched ahead for workers.
|
||
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
|
||
const fresh =
|
||
room && want > 0 && held.length > 0
|
||
? await rows(db.query())
|
||
.where('r.status', 'planned')
|
||
.where('r.spooled_chunks', 0)
|
||
.whereRaw('r.next_id <= r.end_id')
|
||
.whereIn('r.group_id', held)
|
||
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
|
||
.where((q: any) => (leaseCaptureOnly ? q.where('r.capture', true) : q))
|
||
.where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
|
||
.orderByRaw(
|
||
'row_number() OVER (PARTITION BY r.group_id ORDER BY r.start_id), r.priority, r.start_id'
|
||
)
|
||
.limit(want)
|
||
: []
|
||
const targets: Target[] = []
|
||
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
|
||
const start = Number(row.start_id)
|
||
const end = Number(row.end_id)
|
||
const total = Spool.chunksOf({ start, end }).length
|
||
const missing: ChunkSpan[] = []
|
||
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
|
||
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
|
||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
|
||
continue
|
||
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
|
||
missing.push(span)
|
||
}
|
||
if (missing.length > 0) {
|
||
targets.push({
|
||
id: Number(row.id),
|
||
group: row.group,
|
||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||
missing,
|
||
total,
|
||
start,
|
||
end,
|
||
capture: row.capture === true,
|
||
})
|
||
}
|
||
}
|
||
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
|
||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
|
||
targets.push(target)
|
||
return targets
|
||
}
|
||
|
||
private filterRules: { at: number; filter: CorpusFilter } | null = null
|
||
|
||
/** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
|
||
private async corpusFilter(): Promise<CorpusFilter | undefined> {
|
||
if (!coordinatorConfig.corpus.filter) return undefined
|
||
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
|
||
const rules = await RuleSets.current()
|
||
this.filterRules = {
|
||
at: Date.now(),
|
||
filter: { ruleVersion: rules.version, priorities: rules.priorities },
|
||
}
|
||
}
|
||
return this.filterRules.filter
|
||
}
|
||
|
||
/**
|
||
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
|
||
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
|
||
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
|
||
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
|
||
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
|
||
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
|
||
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
|
||
* gets stored_at and isn't read again.
|
||
*/
|
||
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
|
||
const { dir, ahead } = coordinatorConfig.corpus
|
||
if (!dir || ahead <= 0) return []
|
||
const { perRange, perGroup } = coordinatorConfig.spool
|
||
const ofGroup =
|
||
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
|
||
const groups = new Map<string, number>()
|
||
const candidates = await rows(db.query())
|
||
.where('r.capture', true)
|
||
.whereNull('r.stored_at')
|
||
.where((q: any) =>
|
||
q
|
||
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
|
||
.orWhere('r.status', 'done')
|
||
)
|
||
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
|
||
.limit(ahead * 16)
|
||
const targets: Target[] = []
|
||
for (const row of candidates) {
|
||
if (targets.length >= ahead) break
|
||
const id = Number(row.id)
|
||
if (taken.has(id) || this.stored.has(id)) continue
|
||
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
|
||
const start = Number(row.start_id)
|
||
const end = Number(row.end_id)
|
||
const range = { group: row.group, start, end }
|
||
if (await Corpus.complete(range)) {
|
||
this.stored.add(id)
|
||
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
|
||
continue
|
||
}
|
||
const chunks = Spool.chunksOf({ start, end })
|
||
const missing: ChunkSpan[] = []
|
||
for (const span of chunks)
|
||
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
|
||
missing.push(span)
|
||
if (missing.length > 0) {
|
||
targets.push({
|
||
id,
|
||
group: row.group,
|
||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||
missing,
|
||
total: chunks.length,
|
||
start,
|
||
end,
|
||
capture: true,
|
||
corpusOnly: true,
|
||
})
|
||
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
|
||
}
|
||
}
|
||
return targets
|
||
}
|
||
}
|