Daan MeijerandClaude Fable 5.1
137e931e9e
ranges:dates samples each active group's posting dates from the news server (group_dates) and estimates every range's date (ranges.est_date). ranges:capture --from --to marks a window's ranges of every group for the corpus. The corpus fill takes capture ranges in priority order, done ones too, and remembers complete ones (ranges.stored_at). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
81 lines
3.7 KiB
TypeScript
81 lines
3.7 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
|
|
/** Bytes a filtered corpus article takes on disk (334 GB for 18.0B articles, 2026-10-25). */
|
|
const BYTES_PER_ARTICLE = 18.5
|
|
|
|
/**
|
|
* Marks every active group's ranges posted in a window (by ranges.est_date, from ranges:dates) `capture`, so the
|
|
* corpus fill fetches them: voyager 3's test data, a year of all groups at a time (REBUILD.md, corpus). Planned
|
|
* ranges not started get --priority (the fill's order, then the indexing's once there's room); done ones are
|
|
* fetched again into the corpus only. Ranges leased or half spooled are left out (their first chunks would be
|
|
* missing) and reported.
|
|
*/
|
|
export default class RangesCapture extends BaseCommand {
|
|
static commandName = 'ranges:capture'
|
|
static description =
|
|
'Marks the ranges posted in a window for the corpus: --from 2025-01-01 --to 2026-01-01'
|
|
static options: CommandOptions = { startApp: true }
|
|
|
|
@flags.string({ description: 'First day of the window (ISO date)', required: true })
|
|
declare from: string
|
|
|
|
@flags.string({ description: 'Day after the window (ISO date)', required: true })
|
|
declare to: string
|
|
|
|
@flags.number({
|
|
description: 'Priority for the planned ranges marked (default 0: before the history)',
|
|
})
|
|
declare priority?: number
|
|
|
|
@flags.boolean({ description: 'Only report what the window holds' })
|
|
declare dryRun: boolean
|
|
|
|
async run() {
|
|
const from = new Date(this.from)
|
|
const to = new Date(this.to)
|
|
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from)
|
|
return this.logger.error('--from and --to: ISO dates, from before to')
|
|
const priority = this.priority ?? 0
|
|
// A range covers est_date up to the next range's; the last range of a group up to now.
|
|
const hits = `
|
|
WITH spans AS (
|
|
SELECT r.id, r.status, r.capture, r.spooled_chunks, r.end_id - r.start_id + 1 AS n, r.est_date AS d0,
|
|
lead(r.est_date) OVER (PARTITION BY r.group_id ORDER BY r.start_id) AS d1
|
|
FROM ranges r JOIN groups g ON g.id = r.group_id WHERE g.active AND r.est_date IS NOT NULL
|
|
), hit AS (SELECT * FROM spans WHERE d0 < ? AND coalesce(d1, now()) > ?)`
|
|
const { rows } = await db.rawQuery(
|
|
`${hits}
|
|
SELECT CASE WHEN capture THEN 'marked already' WHEN status = 'planned' AND spooled_chunks = 0 THEN 'planned'
|
|
WHEN status = 'done' THEN 'done' ELSE 'in flight' END AS kind,
|
|
count(*) AS ranges, round(sum(n) / 1e9, 2) AS articles
|
|
FROM hit GROUP BY 1 ORDER BY 1`,
|
|
[to, from]
|
|
)
|
|
const fetch = rows
|
|
.filter((r: any) => r.kind === 'planned' || r.kind === 'done')
|
|
.reduce((sum: number, r: any) => sum + Number(r.articles), 0)
|
|
this.logger.info(`${this.from} to ${this.to}:`)
|
|
for (const r of rows)
|
|
this.logger.info(` ${r.kind}: ${r.ranges} ranges, ${r.articles}B articles`)
|
|
this.logger.info(
|
|
`To fetch: ${fetch.toFixed(2)}B articles, ~${((fetch * 1e9 * BYTES_PER_ARTICLE) / 1e12).toFixed(2)} TB in the corpus.`
|
|
)
|
|
if (this.dryRun) return
|
|
const { rows: marked } = await db.rawQuery(
|
|
`${hits}
|
|
UPDATE ranges r SET capture = true,
|
|
priority = CASE WHEN r.status = 'planned' THEN ? ELSE r.priority END
|
|
FROM hit WHERE r.id = hit.id AND NOT hit.capture
|
|
AND (hit.status = 'done' OR (hit.status = 'planned' AND hit.spooled_chunks = 0))
|
|
RETURNING r.status`,
|
|
[to, from, priority]
|
|
)
|
|
const planned = marked.filter((r: any) => r.status === 'planned').length
|
|
this.logger.info(
|
|
`Marked ${planned} planned ranges at priority ${priority} and ${marked.length - planned} done ranges.`
|
|
)
|
|
}
|
|
}
|