daan/usenet-indexer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Posting-time windows into the corpus: ranges:dates, ranges:capture
Test and deploy / test (push) Successful in 55s
Details
Test and deploy / edge (push) Successful in 2m2s
Details
Test and deploy / deploy (push) Successful in 2m12s
Details

Browse Source 
ranges:dates samples each active group's posting dates from the news
server (group_dates) and estimates every range's date (ranges.est_date).
ranges:capture --from --to marks a window's ranges of every group for the
corpus. The corpus fill takes capture ranges in priority order, done ones
too, and remembers complete ones (ranges.stored_at).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 09:47:28 +01:00
1 parent 0469fcd975
commit 137e931e9e
8 files changed
+683 -40
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/prefetcher.ts
+130 -31
View File
@@ -73,7 +73,13 @@ export class Prefetcher {
private writingNow = 0
private sampler?: NodeJS.Timeout
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
private inFlight = new Set<{ rangeId: number; from: number; to: number; since: number; client: XoverClient }>()
private inFlight = new Set<{
rangeId: number
from: number
to: number
since: number
client: XoverClient
}>()
/** Capture ranges found complete in the corpus (not looked at again). */
private stored = new Set<number>()
@@ -163,10 +169,14 @@ export class Prefetcher {
}
const corpusRange = { group: target.group, start: target.start, end: target.end }
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
let data = target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
let data =
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
const stored = data !== null
try {
if (!data) data = target.plain.has(span.from) ? await NntpPool.usePlain(fetch) : await this.fetchHedged(target.id, target.group, span)
if (!data)
data = target.plain.has(span.from)
? await NntpPool.usePlain(fetch)
: await this.fetchHedged(target.id, target.group, span)
} finally {
this.fetchingNow--
}
@@ -176,7 +186,8 @@ export class Prefetcher {
this.stats.chunks++
this.stats.articles += span.to - span.from + 1
this.stats.fetchMs += Date.now() - fetchedAt
if (target.capture) await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
if (target.capture)
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
}
if (target.corpusOnly) {
this.stats.captured++
@@ -193,7 +204,8 @@ export class Prefetcher {
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
[target.id]
)
if (Number(rows[0]?.spooled_chunks) >= target.total) await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
if (Number(rows[0]?.spooled_chunks) >= target.total)
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
} finally {
this.writingNow--
this.stats.writeMs += Date.now() - writtenAt
@@ -215,7 +227,10 @@ export class Prefetcher {
*/
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
let age: number | undefined
const request = (start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null, first: boolean): Request<Buffer> | null => {
const request = (
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
first: boolean
): Request<Buffer> | null => {
let client: XoverClient | undefined
let cancelled = false
const done = start(async (c) => {
@@ -247,7 +262,9 @@ export class Prefetcher {
data = await c.xoverRaw(span.from, span.to)
} catch (error) {
if (!late) throw error
throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`)
throw new Error(
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
)
} finally {
clearInterval(watch)
this.inFlight.delete(entry)
@@ -290,7 +307,8 @@ export class Prefetcher {
/** The longest-running request: range, chunk, how long, how much came in. */
private oldest() {
let oldest: { rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
let oldest:
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
if (!oldest) return 'none in flight'
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
@@ -315,7 +333,12 @@ export class Prefetcher {
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
if (chunks > 0) {
const ages = byAge
.map((b, i) => `${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace('– min', '+ min'))
.map((b, i) =>
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
'– min',
'+ min'
)
)
.join(', ')
logger.info(
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
@@ -341,13 +364,18 @@ export class Prefetcher {
}
private peek(): boolean {
return this.targets.some((t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`)))
return this.targets.some(
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
)
}
/** Whether the target is under its range's and its group's limit of connections. */
private open(target: Target) {
const { perRange, perGroup } = coordinatorConfig.spool
return (perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) && (perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
return (
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
)
}
private count(target: Target, by: number) {
@@ -378,26 +406,58 @@ export class Prefetcher {
this.sizedAt = Date.now()
this.spoolBytes = await Spool.size()
}
const mine = (q: any) => (this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q)
const rows = (q: any) => mine(q.from('ranges as r').join('groups as g', 'g.id', 'r.group_id').select('r.id', 'g.name as group', 'r.start_id', 'r.end_id', 'r.next_id', 'r.plain_chunks', 'r.capture'))
const mine = (q: any) =>
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
const rows = (q: any) =>
mine(
q
.from('ranges as r')
.join('groups as g', 'g.id', 'r.group_id')
.select(
'r.id',
'g.name as group',
'r.start_id',
'r.end_id',
'r.next_id',
'r.plain_chunks',
'r.capture'
)
)
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
const ahead = Math.max(coordinatorConfig.spool.minAhead, coordinatorConfig.spool.aheadPerLease * leased.length)
const started = await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', '>', 0).whereNull('r.spooled_at').orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
const [{ ready }] = await mine(db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)).count('* as ready')
const ahead = Math.max(
coordinatorConfig.spool.minAhead,
coordinatorConfig.spool.aheadPerLease * leased.length
)
const started = await rows(db.query())
.where('r.status', 'planned')
.where('r.spooled_chunks', '>', 0)
.whereNull('r.spooled_at')
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
const [{ ready }] = await mine(
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
).count('* as ready')
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
const { perRange } = coordinatorConfig.spool
const short = perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
const short =
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
const want = Math.max(ahead - Number(ready), short)
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
const full = await CatalogSpace.full()
const fresh = room && want > 0
? await rows(db.query()).where('r.status', 'planned').where('r.spooled_chunks', 0).whereRaw('r.next_id <= r.end_id').where((q: any) => (full ? q.where('r.priority', '<', 0) : q)).orderBy([{ column: 'r.priority' }, { column: 'r.id' }]).limit(want)
: []
const fresh =
room && want > 0
? await rows(db.query())
.where('r.status', 'planned')
.where('r.spooled_chunks', 0)
.whereRaw('r.next_id <= r.end_id')
.where((q: any) => (full ? q.where('r.priority', '<', 0) : q))
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
.limit(want)
: []
const targets: Target[] = []
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
const start = Number(row.start_id)
@@ -405,14 +465,25 @@ export class Prefetcher {
const total = Spool.chunksOf({ start, end }).length
const missing: ChunkSpan[] = []
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span))) missing.push(span)
if (!this.claimed.has(`${row.id}:${span.from}`) && !(await Spool.has(Number(row.id), span)))
missing.push(span)
}
if (missing.length > 0) {
targets.push({ id: Number(row.id), group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total, start, end, capture: row.capture === true })
targets.push({
id: Number(row.id),
group: row.group,
plain: new Set((row.plain_chunks ?? []).map(Number)),
missing,
total,
start,
end,
capture: row.capture === true,
})
}
}
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id)))) targets.push(target)
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
targets.push(target)
return targets
}
@@ -423,23 +494,38 @@ export class Prefetcher {
if (!coordinatorConfig.corpus.filter) return undefined
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
const rules = await RuleSets.current()
this.filterRules = { at: Date.now(), filter: { ruleVersion: rules.version, priorities: rules.priorities } }
this.filterRules = {
at: Date.now(),
filter: { ruleVersion: rules.version, priorities: rules.priorities },
}
}
return this.filterRules.filter
}
/**
* Up to corpus.ahead planned capture ranges, not leased soon, whose chunks aren't all in the corpus; as many of a
* group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which took 4 of the
* loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)).
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). A range found complete
* gets stored_at and isn't read again.
*/
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
const { dir, ahead } = coordinatorConfig.corpus
if (!dir || ahead <= 0) return []
const { perRange, perGroup } = coordinatorConfig.spool
const ofGroup = perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
const ofGroup =
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
const groups = new Map<string, number>()
const candidates = await rows(db.query()).where('r.status', 'planned').where('r.capture', true).where('r.spooled_chunks', 0).orderBy('r.id')
const candidates = await rows(db.query())
.where('r.capture', true)
.whereNull('r.stored_at')
.where((q: any) =>
q
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
.orWhere('r.status', 'done')
)
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
.limit(ahead * 16)
const targets: Target[] = []
for (const row of candidates) {
if (targets.length >= ahead) break
@@ -451,13 +537,26 @@ export class Prefetcher {
const range = { group: row.group, start, end }
if (await Corpus.complete(range)) {
this.stored.add(id)
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
continue
}
const chunks = Spool.chunksOf({ start, end })
const missing: ChunkSpan[] = []
for (const span of chunks) if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span))) missing.push(span)
for (const span of chunks)
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
missing.push(span)
if (missing.length > 0) {
targets.push({ id, group: row.group, plain: new Set((row.plain_chunks ?? []).map(Number)), missing, total: chunks.length, start, end, capture: true, corpusOnly: true })
targets.push({
id,
group: row.group,
plain: new Set((row.plain_chunks ?? []).map(Number)),
missing,
total: chunks.length,
start,
end,
capture: true,
corpusOnly: true,
})
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
}
}