Posting-time windows into the corpus: ranges:dates, ranges:capture
ranges:dates samples each active group's posting dates from the news server (group_dates) and estimates every range's date (ranges.est_date). ranges:capture --from --to marks a window's ranges of every group for the corpus. The corpus fill takes capture ranges in priority order, done ones too, and remembers complete ones (ranges.stored_at). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed
1 parent
0469fcd975
commit
137e931e9e
8 files changed
+683
-40
No files matched your search
@@ -0,0 +1,80 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */
|
||||
const BYTES_PER_ARTICLE = 18.5
|
||||
|
||||
/**
|
||||
* Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the
|
||||
* corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned
|
||||
* ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are
|
||||
* fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be
|
||||
* missing) and reported.
|
||||
*/
|
||||
export default class RangesCapture extends BaseCommand {
|
||||
static commandName = 'ranges:capture'
|
||||
static description =
|
||||
'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'First day of the window (ISO date)', required: true })
|
||||
declare from: string
|
||||
|
||||
@flags.string({ description: 'Day after the window (ISO date)', required: true })
|
||||
declare to: string
|
||||
|
||||
@flags.number({
|
||||
description: 'Priority for the planned ranges marked (default 0: before the history)',
|
||||
})
|
||||
declare priority?: number
|
||||
|
||||
@flags.boolean({ description: 'Only report what the window holds' })
|
||||
declare dryRun: boolean
|
||||
|
||||
async run() {
|
||||
const from = new Date(this.from)
|
||||
const to = new Date(this.to)
|
||||
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from)
|
||||
return this.logger.error('--from and --to: ISO dates, from before to')
|
||||
const priority = this.priority ?? 0
|
||||
// A range covers est_date up to the next range's; the last range of a group up to now.
|
||||
const hits = `
|
||||
WITH spans AS (
|
||||
SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0,
|
||||
lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1
|
||||
FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL
|
||||
), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)`
|
||||
const { rows } = await db.rawQuery(
|
||||
`${hits}
|
||||
SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned'
|
||||
WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind,
|
||||
count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles
|
||||
FROM hit GROUP BY 1 ORDER BY 1`,
|
||||
[to, from]
|
||||
)
|
||||
const fetch = rows
|
||||
.filter((r: any) => r.kind === 'planned' || r.kind === 'done')
|
||||
.reduce((sum: number, r: any) => sum + Number(r.articles), 0)
|
||||
this.logger.info(`${this.from} to ${this.to}:`)
|
||||
for (const r of rows)
|
||||
this.logger.info(` ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`)
|
||||
this.logger.info(
|
||||
`To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.`
|
||||
)
|
||||
if (this.dryRun) return
|
||||
const { rows: marked } = await db.rawQuery(
|
||||
`${hits}
|
||||
UPDATE ranges r SET capture = true,
|
||||
priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END
|
||||
FROM hit WHERE r.id = hit.id AND NOT hit.capture
|
||||
AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0))
|
||||
RETURNING r.status`,
|
||||
[to, from, priority]
|
||||
)
|
||||
const planned = marked.filter((r: any) => r.status === 'planned').length
|
||||
this.logger.info(
|
||||
`Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
|
||||
|
||||
/**
|
||||
* When each active group's articles were posted (group_dates, sampled from the news server) and from that every
|
||||
* range's estimated posting date (ranges.est_date), so ranges can be chosen by era: ranges:capture marks a
|
||||
* posting-time window of every group for the corpus (REBUILD.md, corpus). A group gets a sample every --every
|
||||
* articles (16 … 1000): boneless (161B articles) ~320, three weeks apart; ~15k XOVER commands of 32 articles in all.
|
||||
*/
|
||||
export default class RangesDates extends BaseCommand {
|
||||
static commandName = 'ranges:dates'
|
||||
static description =
|
||||
"Samples when each active group's articles were posted and estimates every range's date"
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@flags.string({ description: 'Only groups whose name matches this SQL LIKE pattern' })
|
||||
declare group?: string
|
||||
|
||||
@flags.number({ description: 'Articles between two samples of a group (default 500M)' })
|
||||
declare every?: number
|
||||
|
||||
@flags.number({
|
||||
description:
|
||||
'Connections, a share of the account beside the coordinator and the prober (default 6)',
|
||||
})
|
||||
declare connections?: number
|
||||
|
||||
@flags.boolean({ description: 'Only groups without samples yet' })
|
||||
declare missing: boolean
|
||||
|
||||
@flags.boolean({
|
||||
description: 'No sampling: estimate the ranges from the samples stored already',
|
||||
})
|
||||
declare estimateOnly: boolean
|
||||
|
||||
async run() {
|
||||
try {
|
||||
if (!this.estimateOnly) await this.sample()
|
||||
await this.estimate()
|
||||
await this.report()
|
||||
} finally {
|
||||
await NntpPool.close()
|
||||
}
|
||||
}
|
||||
|
||||
private groups() {
|
||||
const q = db.from('groups').where('active', true).select('id', 'name').orderBy('priority')
|
||||
if (this.group) q.where('name', 'like', this.group)
|
||||
if (this.missing)
|
||||
q.whereNotExists((sub) =>
|
||||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||||
)
|
||||
return q
|
||||
}
|
||||
|
||||
private async sample() {
|
||||
const every = this.every ?? 500_000_000
|
||||
const connections = this.connections ?? 6
|
||||
const groups = await this.groups()
|
||||
const share = new NntpShare(
|
||||
'dates',
|
||||
coordinatorConfig.nntpShares.coordinatorPriority + 1,
|
||||
1,
|
||||
connections
|
||||
)
|
||||
const first = await NntpPool.share(share)
|
||||
const sharing = setInterval(
|
||||
() =>
|
||||
NntpPool.share(share).catch((error) => this.logger.warning(`NNTP share: ${error.message}`)),
|
||||
coordinatorConfig.nntpShares.everyMs
|
||||
)
|
||||
this.logger.info(
|
||||
`Sampling ${groups.length} groups over ${first.granted} connections, a sample every ${every / 1e6}M articles.`
|
||||
)
|
||||
const startedAt = Date.now()
|
||||
let done = 0
|
||||
let commands = 0
|
||||
let samples = 0
|
||||
const queue = [...groups]
|
||||
const worker = async () => {
|
||||
for (let g = queue.shift(); g; g = queue.shift()) {
|
||||
try {
|
||||
const result = await NntpPool.use((client) =>
|
||||
sampleGroup(client, g.name, { every, size: 32 })
|
||||
)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.from('group_dates').where('group_id', g.id).delete()
|
||||
if (result.samples.length > 0)
|
||||
await trx.table('group_dates').multiInsert(
|
||||
result.samples.map((s) => ({
|
||||
group_id: g.id,
|
||||
article: s.article,
|
||||
posted_at: s.postedAt,
|
||||
}))
|
||||
)
|
||||
})
|
||||
commands += result.commands
|
||||
samples += result.samples.length
|
||||
if (result.samples.length < 2)
|
||||
this.logger.warning(
|
||||
`${g.name}: ${result.samples.length} samples over ${result.low}–${result.high}.`
|
||||
)
|
||||
} catch (error: any) {
|
||||
this.logger.warning(`${g.name}: ${error.message}`)
|
||||
}
|
||||
done++
|
||||
if (done % 50 === 0)
|
||||
this.logger.info(
|
||||
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.max(1, Math.min(connections, groups.length)) }, worker)
|
||||
)
|
||||
} finally {
|
||||
clearInterval(sharing)
|
||||
}
|
||||
this.logger.info(
|
||||
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
|
||||
/** Every range of the groups sampled gets est_date from the group's curve (1000 a statement). */
|
||||
private async estimate() {
|
||||
const groups = await this.groups().whereExists((sub) =>
|
||||
sub.from('group_dates').whereRaw('group_dates.group_id = groups.id')
|
||||
)
|
||||
let updated = 0
|
||||
for (const g of groups) {
|
||||
const rows = await db
|
||||
.from('group_dates')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('article')
|
||||
.select('article', 'posted_at')
|
||||
const samples: DateSample[] = rows.map((r) => ({
|
||||
article: Number(r.article),
|
||||
postedAt: new Date(r.posted_at),
|
||||
}))
|
||||
const ranges = await db
|
||||
.from('ranges')
|
||||
.where('group_id', g.id)
|
||||
.orderBy('start_id')
|
||||
.select('id', 'start_id')
|
||||
for (let i = 0; i < ranges.length; i += 1000) {
|
||||
const batch = ranges.slice(i, i + 1000)
|
||||
const values = batch.map(() => '(?::bigint, ?::timestamptz)').join(', ')
|
||||
const bindings = batch.flatMap((r) => [
|
||||
Number(r.id),
|
||||
estimateDate(samples, Number(r.start_id))!.toISOString(),
|
||||
])
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges AS r SET est_date = v.d FROM (VALUES ${values}) AS v(id, d) WHERE r.id = v.id`,
|
||||
bindings
|
||||
)
|
||||
updated += batch.length
|
||||
}
|
||||
}
|
||||
this.logger.info(`${updated} ranges of ${groups.length} groups dated.`)
|
||||
}
|
||||
|
||||
/** The planned history by posting year: what a window would take. */
|
||||
private async report() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
SELECT extract(year FROM est_date)::int AS year, count(*) AS ranges, count(DISTINCT group_id) AS groups,
|
||||
round(sum(end_id - start_id + 1) / 1e9, 1) AS articles,
|
||||
round(sum(end_id - start_id + 1) FILTER (WHERE status = 'done') / 1e9, 1) AS done,
|
||||
round(sum(end_id - start_id + 1) FILTER (WHERE capture) / 1e9, 1) AS capture
|
||||
FROM ranges WHERE est_date IS NOT NULL GROUP BY 1 ORDER BY 1`)
|
||||
const undatedRows = await db.rawQuery(
|
||||
`SELECT count(*) AS undated FROM ranges WHERE est_date IS NULL`
|
||||
)
|
||||
const undated = undatedRows.rows[0].undated
|
||||
this.logger.info(
|
||||
`By estimated posting year (B articles; done: indexed, capture: marked for the corpus); ${undated} ranges undated:`
|
||||
)
|
||||
for (const r of rows)
|
||||
this.logger.info(
|
||||
` ${r.year}: ${String(r.ranges).padStart(7)} ranges of ${String(r.groups).padStart(3)} groups, ${String(r.articles).padStart(6)}B articles, ${r.done}B done, ${r.capture}B capture`
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user