Daan MeijerandClaude Opus 5.5
cfce8884d4
One article of a multi-part file under a name like "nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr" or "bslXlwgPiqZadhRa" (a random subject and poster per article, with fake counters) was a release: looksRandom misses these names for their separators. 160k of alt.binaries.storage's 162k non-Bob releases, and about 60% of all single-article releases in a 2% sample. looks_scrambled / isScrambledFile: at least half the stem's letters and digits in tokens of 5+ that switch case or letter/digit 3 times or more (93% of the storage spam). Used for lone articles given up before the block's end only; at the end the strict looksRandom stays, as the rest of such a file can be in the next block. The namer's re-index (partialFiles) does the same. The prefetcher's corpus fill skips groups that aren't active. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
611 lines
25 KiB
TypeScript
611 lines
25 KiB
TypeScript
import db from '@adonisjs/lucid/services/db'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import NntpPool from '#services/nntp_pool'
|
||
import Spool, { type ChunkSpan } from '#services/spool'
|
||
import Corpus, { type CorpusFilter } from '#services/corpus'
|
||
import RuleSets from '#services/rule_sets'
|
||
import CatalogSpace from '#services/catalog_space'
|
||
import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
|
||
|
||
interface Target {
|
||
id: number
|
||
group: string
|
||
/** Chunk starts to fetch without compression (ranges.plain_chunks). */
|
||
plain: Set<number>
|
||
/** Chunks not spooled yet, in order. */
|
||
missing: ChunkSpan[]
|
||
/** All of the range's chunks, from its start. */
|
||
total: number
|
||
start: number
|
||
end: number
|
||
/** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
|
||
capture: boolean
|
||
/** Fetched into the corpus only, not the spool: a capture range no worker leases soon (corpus.ahead). */
|
||
corpusOnly?: boolean
|
||
}
|
||
|
||
/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
|
||
const STALL_SAMPLE_MS = 15_000
|
||
const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS
|
||
|
||
/** Connection ages the report counts slow first requests by, in minutes. */
|
||
const AGE_BUCKETS = [2, 10, Infinity]
|
||
|
||
/** How often the targets are read again from the database. */
|
||
const REFRESH_MS = 2000
|
||
/** How often the spool's size is measured (a stat per file). */
|
||
const SIZE_MS = 30_000
|
||
|
||
/**
|
||
* Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
|
||
* Every connection takes the next missing chunk: the leased ranges' first
|
||
* (a worker waits for them), then the next planned ranges, with at most
|
||
* PREFETCH_PER_RANGE connections on a range and PREFETCH_PER_GROUP on a
|
||
* group (Eweka answers faster spread out), about SPOOL_AHEAD_PER_LEASE per
|
||
* active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
|
||
* connection fills ~26 k overview lines/s, so a range on one connection
|
||
* kept its worker waiting (LOG.md 2026-10-09). A chunk that takes long is
|
||
* asked again on a spare connection (fetchHedged).
|
||
*/
|
||
export class Prefetcher {
|
||
/**
|
||
* This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
|
||
* process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
|
||
* LOG.md 2026-10-09), less than 50 connections bring.
|
||
*/
|
||
constructor(private shard = { index: 0, count: 1 }) {}
|
||
|
||
private targets: Target[] = []
|
||
private claimed = new Set<string>()
|
||
/** Chunks being fetched per range and per group (spool.perRange, spool.perGroup). */
|
||
private perRange = new Map<number, number>()
|
||
private perGroup = new Map<string, number>()
|
||
private running = 0
|
||
private refreshedAt = 0
|
||
private stopped = false
|
||
private spoolBytes = 0
|
||
private sizedAt = 0
|
||
/** Fetched since the last report: chunks, articles, ms spent fetching. */
|
||
private stats = this.freshStats()
|
||
/** Loops in a fetch or a spool write right now. */
|
||
private fetchingNow = 0
|
||
private writingNow = 0
|
||
private sampler?: NodeJS.Timeout
|
||
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
|
||
private inFlight = new Set<{
|
||
rangeId: number
|
||
from: number
|
||
to: number
|
||
since: number
|
||
client: XoverClient
|
||
}>()
|
||
/** Capture ranges found complete in the corpus (not looked at again). */
|
||
private stored = new Set<number>()
|
||
|
||
private freshStats() {
|
||
return {
|
||
chunks: 0,
|
||
/** Chunks read from the corpus instead of fetched. */
|
||
stored: 0,
|
||
/** Chunks fetched into the corpus only (corpus.ahead). */
|
||
captured: 0,
|
||
articles: 0,
|
||
fetchMs: 0,
|
||
hedged: 0,
|
||
hedgeWon: 0,
|
||
/** First requests per connection-age bucket, and how many of them were slow. */
|
||
byAge: AGE_BUCKETS.map(() => ({ requests: 0, slow: 0 })),
|
||
/** Spool writes and the range's update, ms. */
|
||
writeMs: 0,
|
||
/** Per answered request, ms: GROUP, XOVER until its status line, the transfer; transfers with a silence over 5 s. */
|
||
requests: 0,
|
||
groupMs: 0,
|
||
replyMs: 0,
|
||
bodyMs: 0,
|
||
silent: 0,
|
||
/** Once a second: loops fetching, writing, not running; pool connections with a command in flight. */
|
||
samples: 0,
|
||
fetching: 0,
|
||
writing: 0,
|
||
idle: 0,
|
||
inFlight: 0,
|
||
inFlightMin: Infinity,
|
||
/** Requests given up as stalled (nntpStallBytesPerMinute). */
|
||
deadlines: 0,
|
||
ranges: 0,
|
||
groups: 0,
|
||
since: Date.now(),
|
||
}
|
||
}
|
||
|
||
/** The prefetcher's loops: the pool less the connections kept for hedging. */
|
||
private get loops() {
|
||
return Math.max(1, NntpPool.size - coordinatorConfig.nntpHedgeConnections)
|
||
}
|
||
|
||
/** One round: refreshes the targets now and then, and puts idle connections to work. */
|
||
async tick() {
|
||
this.sampler ??= setInterval(() => this.sample(), 1000).unref()
|
||
if (Date.now() - this.refreshedAt >= REFRESH_MS) {
|
||
this.refreshedAt = Date.now()
|
||
this.targets = await this.readTargets()
|
||
}
|
||
this.report()
|
||
while (!this.stopped && this.running < this.loops && this.peek()) {
|
||
this.running++
|
||
this.work()
|
||
.catch((error) => logger.warn(`Prefetching: ${error.message}`))
|
||
.finally(() => this.running--)
|
||
}
|
||
// Work left with every loop running: the pool's share could be bigger.
|
||
if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
|
||
}
|
||
|
||
stop() {
|
||
this.stopped = true
|
||
clearInterval(this.sampler)
|
||
}
|
||
|
||
get active() {
|
||
return this.running
|
||
}
|
||
|
||
/** One connection's loop: chunks until there is nothing left to fetch. */
|
||
private async work() {
|
||
while (!this.stopped) {
|
||
// The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
|
||
// Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
|
||
if (this.running > this.loops) return
|
||
const next = this.take()
|
||
if (!next) return
|
||
const { target, span } = next
|
||
try {
|
||
const fetchedAt = Date.now()
|
||
this.fetchingNow++
|
||
const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
|
||
await client.group(target.group)
|
||
return client.xoverRaw(span.from, span.to)
|
||
}
|
||
const corpusRange = { group: target.group, start: target.start, end: target.end }
|
||
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
|
||
let data =
|
||
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||
const stored = data !== null
|
||
try {
|
||
if (!data)
|
||
data = target.plain.has(span.from)
|
||
? await NntpPool.usePlain(fetch)
|
||
: await this.fetchHedged(target.id, target.group, span)
|
||
} finally {
|
||
this.fetchingNow--
|
||
}
|
||
if (stored) {
|
||
this.stats.stored++
|
||
} else {
|
||
this.stats.chunks++
|
||
this.stats.articles += span.to - span.from + 1
|
||
this.stats.fetchMs += Date.now() - fetchedAt
|
||
if (target.capture)
|
||
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
|
||
}
|
||
if (target.corpusOnly) {
|
||
this.stats.captured++
|
||
continue
|
||
}
|
||
const writtenAt = Date.now()
|
||
this.writingNow++
|
||
try {
|
||
const existed = await Spool.has(target.id, span)
|
||
await Spool.write(target.id, span, data)
|
||
if (existed) continue
|
||
this.spoolBytes += data.length
|
||
const { rows } = await db.rawQuery(
|
||
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
|
||
[target.id]
|
||
)
|
||
if (Number(rows[0]?.spooled_chunks) >= target.total)
|
||
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||
} finally {
|
||
this.writingNow--
|
||
this.stats.writeMs += Date.now() - writtenAt
|
||
}
|
||
} finally {
|
||
this.claimed.delete(`${target.id}:${span.from}`)
|
||
this.count(target, -1)
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* A chunk over a pooled connection. Not in after nntpHedgeAfterMs, it's
|
||
* asked again on a free connection (if there is one): Eweka stalls single
|
||
* requests, and the same chunk on another connection mostly comes in
|
||
* seconds (LOG.md 2026-10-09 (15)). The first answer wins; the other
|
||
* request can't be cancelled, so its connection is closed (the pool opens
|
||
* a fresh one).
|
||
*/
|
||
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
|
||
let age: number | undefined
|
||
const request = (
|
||
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
|
||
first: boolean
|
||
): Request<Buffer> | null => {
|
||
let client: XoverClient | undefined
|
||
let cancelled = false
|
||
const done = start(async (c) => {
|
||
// Given up before it got a connection: the connection goes back unused.
|
||
if (cancelled) return Buffer.alloc(0)
|
||
client = c
|
||
if (first) age = NntpPool.ageOf(c)
|
||
const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
|
||
this.inFlight.add(entry)
|
||
// Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall.
|
||
let late = false
|
||
const samples: number[] = []
|
||
const watch = setInterval(() => {
|
||
samples.push(c.buffered)
|
||
if (samples.length <= STALL_SAMPLES) return
|
||
const minuteAgo = samples.shift()!
|
||
if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) {
|
||
late = true
|
||
this.stats.deadlines++
|
||
c.close()
|
||
}
|
||
}, STALL_SAMPLE_MS)
|
||
let data: Buffer
|
||
let groupMs = 0
|
||
try {
|
||
const groupAt = performance.now()
|
||
await c.group(group)
|
||
groupMs = performance.now() - groupAt
|
||
data = await c.xoverRaw(span.from, span.to)
|
||
} catch (error) {
|
||
if (!late) throw error
|
||
throw new Error(
|
||
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
|
||
)
|
||
} finally {
|
||
clearInterval(watch)
|
||
this.inFlight.delete(entry)
|
||
}
|
||
if (!cancelled) {
|
||
const t = c.lastTiming
|
||
this.stats.requests++
|
||
this.stats.groupMs += groupMs
|
||
this.stats.replyMs += t.replyMs
|
||
this.stats.bodyMs += t.bodyMs
|
||
if (t.longestGapMs > 5000) this.stats.silent++
|
||
}
|
||
return data
|
||
})
|
||
if (!done) return null
|
||
done.catch(() => {})
|
||
return {
|
||
done,
|
||
// The request can't be called off on the wire: its connection is closed, and the pool opens a fresh one.
|
||
cancel: () => {
|
||
cancelled = true
|
||
client?.close()
|
||
},
|
||
}
|
||
}
|
||
const result = await hedge(
|
||
() => request((work) => NntpPool.use(work), true)!,
|
||
() => request((work) => NntpPool.tryUse(work), false),
|
||
coordinatorConfig.nntpHedgeAfterMs
|
||
)
|
||
if (age !== undefined) {
|
||
const bucket = this.stats.byAge[AGE_BUCKETS.findIndex((minutes) => age! < minutes * 60_000)]
|
||
bucket.requests++
|
||
if (result.late) bucket.slow++
|
||
}
|
||
if (result.hedged) this.stats.hedged++
|
||
if (result.secondWon) this.stats.hedgeWon++
|
||
return result.value
|
||
}
|
||
|
||
/** The longest-running request: range, chunk, how long, how much came in. */
|
||
private oldest() {
|
||
let oldest:
|
||
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
|
||
if (!oldest) return 'none in flight'
|
||
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
|
||
}
|
||
|
||
private sample() {
|
||
const s = this.stats
|
||
s.samples++
|
||
s.fetching += this.fetchingNow
|
||
s.writing += this.writingNow
|
||
s.idle += Math.max(0, this.loops - this.fetchingNow - this.writingNow)
|
||
s.inFlight += NntpPool.busyCount
|
||
s.inFlightMin = Math.min(s.inFlightMin, NntpPool.busyCount)
|
||
s.ranges += this.perRange.size
|
||
s.groups += this.perGroup.size
|
||
}
|
||
|
||
/** A line a minute: what the connections fetched, and how long a chunk took. */
|
||
private report() {
|
||
const seconds = (Date.now() - this.stats.since) / 1000
|
||
if (seconds < 60) return
|
||
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
|
||
if (chunks > 0) {
|
||
const ages = byAge
|
||
.map((b, i) =>
|
||
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
|
||
'– min',
|
||
'+ min'
|
||
)
|
||
)
|
||
.join(', ')
|
||
logger.info(
|
||
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
|
||
`${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
|
||
`hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
|
||
(this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus` : '') +
|
||
(this.stats.captured > 0 ? `; ${this.stats.captured} fetched into the corpus only` : '') +
|
||
'.'
|
||
)
|
||
const st = this.stats
|
||
const avg = (n: number) => (samples > 0 ? n / samples : 0)
|
||
const per = (ms: number) => (requests > 0 ? (ms / requests / 1000).toFixed(1) : '-')
|
||
logger.info(
|
||
`Connections: ${avg(st.inFlight).toFixed(1)} of ${NntpPool.size} with a command in flight (min ${st.inFlightMin}), ` +
|
||
`on ${avg(st.ranges).toFixed(1)} ranges of ${avg(st.groups).toFixed(1)} groups; ` +
|
||
`loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` +
|
||
`a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` +
|
||
`${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` +
|
||
`${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.`
|
||
)
|
||
}
|
||
this.stats = this.freshStats()
|
||
}
|
||
|
||
private peek(): boolean {
|
||
return this.targets.some(
|
||
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
|
||
)
|
||
}
|
||
|
||
/** Whether the target is under its range's and its group's limit of connections. */
|
||
private open(target: Target) {
|
||
const { perRange, perGroup } = coordinatorConfig.spool
|
||
return (
|
||
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
|
||
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||
)
|
||
}
|
||
|
||
private count(target: Target, by: number) {
|
||
const range = (this.perRange.get(target.id) ?? 0) + by
|
||
const group = (this.perGroup.get(target.group) ?? 0) + by
|
||
if (range > 0) this.perRange.set(target.id, range)
|
||
else this.perRange.delete(target.id)
|
||
if (group > 0) this.perGroup.set(target.group, group)
|
||
else this.perGroup.delete(target.group)
|
||
}
|
||
|
||
/** The first unclaimed missing chunk, in target order, of a target under its limits. */
|
||
private take(): { target: Target; span: ChunkSpan } | null {
|
||
for (const target of this.targets) {
|
||
if (!this.open(target)) continue
|
||
const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
|
||
if (index === -1) continue
|
||
const [span] = target.missing.splice(index, 1)
|
||
this.claimed.add(`${target.id}:${span.from}`)
|
||
this.count(target, 1)
|
||
return { target, span }
|
||
}
|
||
return null
|
||
}
|
||
|
||
private async readTargets(): Promise<Target[]> {
|
||
if (Date.now() - this.sizedAt >= SIZE_MS) {
|
||
this.sizedAt = Date.now()
|
||
this.spoolBytes = await Spool.size()
|
||
}
|
||
const mine = (q: any) =>
|
||
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
|
||
const rows = (q: any) =>
|
||
mine(
|
||
q
|
||
.from('ranges as r')
|
||
.join('groups as g', 'g.id', 'r.group_id')
|
||
.select(
|
||
'r.id',
|
||
'g.name as group',
|
||
'r.start_id',
|
||
'r.end_id',
|
||
'r.next_id',
|
||
'r.plain_chunks',
|
||
'r.capture'
|
||
)
|
||
)
|
||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||
// The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
|
||
// cursors (a group's history and tail lanes are two cursors).
|
||
const held: { group: number; lane: string }[] = (
|
||
await db
|
||
.from('group_leases')
|
||
.whereNotNull('node_id')
|
||
.where(
|
||
'heartbeat_at',
|
||
'>=',
|
||
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
|
||
)
|
||
.select('group_id', 'lane')
|
||
).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
|
||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||
const ahead = Math.max(
|
||
coordinatorConfig.spool.minAhead,
|
||
coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
|
||
)
|
||
const started = await rows(db.query())
|
||
.where('r.status', 'planned')
|
||
.where('r.spooled_chunks', '>', 0)
|
||
.whereNull('r.spooled_at')
|
||
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||
const [{ ready }] = await mine(
|
||
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
|
||
).count('* as ready')
|
||
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
|
||
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
|
||
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
|
||
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
|
||
const { perRange } = coordinatorConfig.spool
|
||
const short =
|
||
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||
const want = Math.max(ahead - Number(ready), short)
|
||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||
const full = await CatalogSpace.full()
|
||
// Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
|
||
// leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
|
||
// at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
|
||
// window worker running) nothing is fetched ahead for workers.
|
||
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
|
||
const tailPriority = coordinatorConfig.tail.priority
|
||
const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
|
||
const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
|
||
const fresh =
|
||
room && want > 0 && heldHistory.length + heldTail.length > 0
|
||
? await rows(db.query())
|
||
.where('r.status', 'planned')
|
||
.where('r.spooled_chunks', 0)
|
||
.whereRaw('r.next_id <= r.end_id')
|
||
.where((q: any) =>
|
||
q
|
||
.where((h: any) =>
|
||
h
|
||
.whereIn('r.group_id', heldHistory)
|
||
.where('r.priority', '>', tailPriority)
|
||
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
|
||
.where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
|
||
)
|
||
.orWhere((t: any) =>
|
||
t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
|
||
)
|
||
)
|
||
.orderByRaw(
|
||
'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
|
||
[tailPriority]
|
||
)
|
||
.limit(want)
|
||
: []
|
||
const targets: Target[] = []
|
||
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
|
||
const start = Number(row.start_id)
|
||
const end = Number(row.end_id)
|
||
const total = Spool.chunksOf({ start, end }).length
|
||
const missing: ChunkSpan[] = []
|
||
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
|
||
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
|
||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
|
||
continue
|
||
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
|
||
missing.push(span)
|
||
}
|
||
if (missing.length > 0) {
|
||
targets.push({
|
||
id: Number(row.id),
|
||
group: row.group,
|
||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||
missing,
|
||
total,
|
||
start,
|
||
end,
|
||
capture: row.capture === true,
|
||
})
|
||
}
|
||
}
|
||
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
|
||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
|
||
targets.push(target)
|
||
return targets
|
||
}
|
||
|
||
private filterRules: { at: number; filter: CorpusFilter } | null = null
|
||
|
||
/** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
|
||
private async corpusFilter(): Promise<CorpusFilter | undefined> {
|
||
if (!coordinatorConfig.corpus.filter) return undefined
|
||
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
|
||
const rules = await RuleSets.current()
|
||
this.filterRules = {
|
||
at: Date.now(),
|
||
filter: { ruleVersion: rules.version, priorities: rules.priorities },
|
||
}
|
||
}
|
||
return this.filterRules.filter
|
||
}
|
||
|
||
/**
|
||
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
|
||
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
|
||
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
|
||
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
|
||
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
|
||
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
|
||
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
|
||
* gets stored_at and isn't read again.
|
||
*/
|
||
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
|
||
const { dir, ahead } = coordinatorConfig.corpus
|
||
if (!dir || ahead <= 0) return []
|
||
const { perRange, perGroup } = coordinatorConfig.spool
|
||
const ofGroup =
|
||
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
|
||
const groups = new Map<string, number>()
|
||
const candidates = await rows(db.query())
|
||
.where('r.capture', true)
|
||
// A group taken off the list (groups.active) isn't worth the connections.
|
||
.where('g.active', true)
|
||
.whereNull('r.stored_at')
|
||
.where((q: any) =>
|
||
q
|
||
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
|
||
.orWhere('r.status', 'done')
|
||
)
|
||
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
|
||
.limit(ahead * 16)
|
||
const targets: Target[] = []
|
||
for (const row of candidates) {
|
||
if (targets.length >= ahead) break
|
||
const id = Number(row.id)
|
||
if (taken.has(id) || this.stored.has(id)) continue
|
||
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
|
||
const start = Number(row.start_id)
|
||
const end = Number(row.end_id)
|
||
const range = { group: row.group, start, end }
|
||
if (await Corpus.complete(range)) {
|
||
this.stored.add(id)
|
||
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
|
||
continue
|
||
}
|
||
const chunks = Spool.chunksOf({ start, end })
|
||
const missing: ChunkSpan[] = []
|
||
for (const span of chunks)
|
||
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
|
||
missing.push(span)
|
||
if (missing.length > 0) {
|
||
targets.push({
|
||
id,
|
||
group: row.group,
|
||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||
missing,
|
||
total: chunks.length,
|
||
start,
|
||
end,
|
||
capture: true,
|
||
corpusOnly: true,
|
||
})
|
||
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
|
||
}
|
||
}
|
||
return targets
|
||
}
|
||
}
|