Daan MeijerandClaude Fable 5.1
3cc7d9738e
Eweka stops a compressed XOVER reply at the first ".\r\n" in its own output (78 01 2e 0d 0a and nothing more), which some small replies begin with; the text path waited for more until the reply timeout. ranges:dates uses plain connections of its own instead of the pool. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
217 lines
8.3 KiB
TypeScript
217 lines
8.3 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||
import db from '@adonisjs/lucid/services/db'
|
||
import env from '#start/env'
|
||
import coordinatorConfig from '#config/coordinator'
|
||
import { XoverClient } from '#lib/nntp/index'
|
||
import { NntpShare } from '#services/nntp_shares'
|
||
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
|
||
|
||
/**
|
||
* When each active group's articles were posted (group_dates, sampled from the news server) and from that every
|
||
* range's estimated posting date (ranges.est_date), so ranges can be chosen by era: ranges:capture marks a
|
||
* posting-time window of every group for the corpus (REBUILD.md, corpus). A group gets a sample every --every
|
||
* articles (16 … 1000): boneless (161B articles) ~320, three weeks apart; ~15k XOVER commands of 32 articles in all.
|
||
*/
|
||
export default class RangesDates extends BaseCommand {
|
||
static commandName = 'ranges:dates'
|
||
static description =
|
||
"Samples when each active group's articles were posted and estimates every range's date"
|
||
static options: CommandOptions = { startApp: true }
|
||
|
||
@flags.string({ description: 'Only groups whose name matches this SQL LIKE pattern' })
|
||
declare group?: string
|
||
|
||
@flags.number({ description: 'Articles between two samples of a group (default 500M)' })
|
||
declare every?: number
|
||
|
||
@flags.number({
|
||
description:
|
||
'Connections, a share of the account beside the coordinator and the prober (default 6)',
|
||
})
|
||
declare connections?: number
|
||
|
||
@flags.boolean({ description: 'Only groups without samples yet' })
|
||
declare missing: boolean
|
||
|
||
@flags.boolean({
|
||
description: 'No sampling: estimate the ranges from the samples stored already',
|
||
})
|
||
declare estimateOnly: boolean
|
||
|
||
async run() {
|
||
if (!this.estimateOnly) await this.sample()
|
||
await this.estimate()
|
||
await this.report()
|
||
}
|
||
|
||
private groups(missing = this.missing) {
|
||
const q = db.from('groups').where('active', true).select('id', 'name').orderBy('priority')
|
||
if (this.group) q.where('name', 'like', this.group)
|
||
if (missing)
|
||
q.whereNotExists((sub) =>
|
||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||
)
|
||
return q
|
||
}
|
||
|
||
private async sample() {
|
||
const every = this.every ?? 500_000_000
|
||
const connections = this.connections ?? 6
|
||
const groups = await this.groups()
|
||
// Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||
// output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
|
||
// answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
|
||
// room for them), as many as asked for.
|
||
const workers = Math.max(1, Math.min(connections, groups.length))
|
||
const share = new NntpShare(
|
||
'dates',
|
||
coordinatorConfig.nntpShares.coordinatorPriority + 1,
|
||
workers,
|
||
workers
|
||
)
|
||
let open = 0
|
||
const sync = () =>
|
||
share
|
||
.sync(workers, open)
|
||
.catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
|
||
await sync()
|
||
const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
|
||
this.logger.info(
|
||
`Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
|
||
)
|
||
const startedAt = Date.now()
|
||
let done = 0
|
||
let commands = 0
|
||
let samples = 0
|
||
const queue = [...groups]
|
||
const worker = async () => {
|
||
let client: XoverClient | null = null
|
||
const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
|
||
if (!client) {
|
||
client = await XoverClient.connect({
|
||
host: env.get('NNTP_HOST')!,
|
||
port: env.get('NNTP_PORT', 563),
|
||
user: env.get('NNTP_USER'),
|
||
password: env.get('NNTP_PASSWORD'),
|
||
secure: env.get('NNTP_SECURE', true),
|
||
compress: false,
|
||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||
})
|
||
open++
|
||
}
|
||
try {
|
||
return await work(client)
|
||
} catch (error) {
|
||
client.close()
|
||
client = null
|
||
open--
|
||
throw error
|
||
}
|
||
}
|
||
for (let g = queue.shift(); g; g = queue.shift()) {
|
||
try {
|
||
const result = await sampleGroup(use, g.name, { every, size: 32 })
|
||
for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
|
||
await db.transaction(async (trx) => {
|
||
await trx.from('group_dates').where('group_id', g.id).delete()
|
||
if (result.samples.length > 0)
|
||
await trx.table('group_dates').multiInsert(
|
||
result.samples.map((s) => ({
|
||
group_id: g.id,
|
||
article: s.article,
|
||
posted_at: s.postedAt,
|
||
}))
|
||
)
|
||
})
|
||
commands += result.commands
|
||
samples += result.samples.length
|
||
if (result.samples.length < 2)
|
||
this.logger.warning(
|
||
`${g.name}: ${result.samples.length} samples over ${result.low}–${result.high}.`
|
||
)
|
||
} catch (error: any) {
|
||
this.logger.warning(`${g.name}: ${error.message}`)
|
||
}
|
||
done++
|
||
if (done % 50 === 0)
|
||
this.logger.info(
|
||
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||
)
|
||
}
|
||
if (client) {
|
||
;(client as XoverClient).close()
|
||
open--
|
||
}
|
||
}
|
||
try {
|
||
await Promise.all(Array.from({ length: workers }, worker))
|
||
} finally {
|
||
clearInterval(sharing)
|
||
await share.sync(0, 0).catch(() => {})
|
||
}
|
||
this.logger.info(
|
||
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||
)
|
||
}
|
||
|
||
/** Every range of the groups sampled gets est_date from the group's curve (1000 a statement). */
|
||
private async estimate() {
|
||
const groups = await this.groups(false).whereExists((sub) =>
|
||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||
)
|
||
let updated = 0
|
||
for (const g of groups) {
|
||
const rows = await db
|
||
.from('group_dates')
|
||
.where('group_id', g.id)
|
||
.orderBy('article')
|
||
.select('article', 'posted_at')
|
||
const samples: DateSample[] = rows.map((r) => ({
|
||
article: Number(r.article),
|
||
postedAt: new Date(r.posted_at),
|
||
}))
|
||
const ranges = await db
|
||
.from('ranges')
|
||
.where('group_id', g.id)
|
||
.orderBy('start_id')
|
||
.select('id', 'start_id')
|
||
for (let i = 0; i < ranges.length; i += 1000) {
|
||
const batch = ranges.slice(i, i + 1000)
|
||
const values = batch.map(() => '(?::bigint, ?::timestamptz)').join(', ')
|
||
const bindings = batch.flatMap((r) => [
|
||
Number(r.id),
|
||
estimateDate(samples, Number(r.start_id))!.toISOString(),
|
||
])
|
||
await db.rawQuery(
|
||
`UPDATE ranges AS r SET est_date = v.d FROM (VALUES ${values}) AS v(id, d) WHERE r.id = v.id`,
|
||
bindings
|
||
)
|
||
updated += batch.length
|
||
}
|
||
}
|
||
this.logger.info(`${updated} ranges of ${groups.length} groups dated.`)
|
||
}
|
||
|
||
/** The planned history by posting year: what a window would take. */
|
||
private async report() {
|
||
const { rows } = await db.rawQuery(`
|
||
SELECT extract(year FROM est_date)::int AS year, count(*) AS ranges, count(DISTINCT group_id) AS groups,
|
||
round(sum(end_id - start_id + 1) / 1e9, 1) AS articles,
|
||
round(sum(end_id - start_id + 1) FILTER (WHERE status = 'done') / 1e9, 1) AS done,
|
||
round(sum(end_id - start_id + 1) FILTER (WHERE capture) / 1e9, 1) AS capture
|
||
FROM ranges WHERE est_date IS NOT NULL GROUP BY 1 ORDER BY 1`)
|
||
const undatedRows = await db.rawQuery(
|
||
`SELECT count(*) AS undated FROM ranges WHERE est_date IS NULL`
|
||
)
|
||
const undated = undatedRows.rows[0].undated
|
||
this.logger.info(
|
||
`By estimated posting year (B articles; done: indexed, capture: marked for the corpus); ${undated} ranges undated:`
|
||
)
|
||
for (const r of rows)
|
||
this.logger.info(
|
||
` ${r.year}: ${String(r.ranges).padStart(7)} ranges of ${String(r.groups).padStart(3)} groups, ${String(r.articles).padStart(6)}B articles, ${r.done}B done, ${r.capture}B capture`
|
||
)
|
||
}
|
||
}
|