import { BaseCommand, flags } from '@adonisjs/core/ace' import type { CommandOptions } from '@adonisjs/core/types/ace' import db from '@adonisjs/lucid/services/db' /** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */ const BYTES_PER_ARTICLE = 18.5 /** * Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the * corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned * ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are * fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be * missing) and reported. */ export default class RangesCapture extends BaseCommand { static commandName = 'ranges:capture' static description = 'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01' static options: CommandOptions = { startApp: true } @flags.string({ description: 'First day of the window (ISO date)', required: true }) declare from: string @flags.string({ description: 'Day after the window (ISO date)', required: true }) declare to: string @flags.number({ description: 'Priority for the planned ranges marked (default 0: before the history)', }) declare priority?: number @flags.boolean({ description: 'Only report what the window holds' }) declare dryRun: boolean async run() { const from = new Date(this.from) const to = new Date(this.to) if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from) return this.logger.error('--from and --to: ISO dates, from before to') const priority = this.priority ?? 0 // A range covers est_date up to the next range's; the last range of a group up to now. const hits = ` WITH spans AS ( SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0, lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1 FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL ), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)` const { rows } = await db.rawQuery( `${hits} SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned' WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind, count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles FROM hit GROUP BY 1 ORDER BY 1`, [to, from] ) const fetch = rows .filter((r: any) => r.kind === 'planned' || r.kind === 'done') .reduce((sum: number, r: any) => sum + Number(r.articles), 0) this.logger.info(`${this.from} to ${this.to}:`) for (const r of rows) this.logger.info(` ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`) this.logger.info( `To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.` ) if (this.dryRun) return const { rows: marked } = await db.rawQuery( `${hits} UPDATE ranges r SET capture = true, priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END FROM hit WHERE r.id = hit.id AND NOT hit.capture AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0)) RETURNING r.status`, [to, from, priority] ) const planned = marked.filter((r: any) => r.status === 'planned').length this.logger.info( `Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.` ) } }