Daan MeijerandClaude Opus 5.5
be8a3d4580
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
53 lines
1.9 KiB
TypeScript
53 lines
1.9 KiB
TypeScript
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||
import db from '@adonisjs/lucid/services/db'
|
||
import NntpService from '#services/NntpService'
|
||
import BackfillService from '#services/BackfillService'
|
||
import { isSpotGroup } from '#services/SpotService'
|
||
|
||
/**
|
||
* Indexes a group's older headers (CONCEPT.md D32): from where live
|
||
* indexing began back to N days ago, header-only and behind live indexing.
|
||
* index:scheduler queues the ranges; this only sets where to go.
|
||
*/
|
||
export default class GroupBackfill extends BaseCommand {
|
||
public static commandName = 'group:backfill'
|
||
public static description = "Indexes groups' older headers (header-only, behind live indexing)."
|
||
|
||
public static options: CommandOptions = {
|
||
startApp: true,
|
||
}
|
||
|
||
@args.spread({ description: 'Groups (default: every active binary group)', required: false })
|
||
declare names?: string[]
|
||
|
||
@flags.number({ description: 'How many days back', default: 21 })
|
||
declare days: number
|
||
|
||
@flags.boolean({ description: 'Stop backfilling these groups (ranges already queued still run)' })
|
||
declare stop: boolean
|
||
|
||
public async run() {
|
||
const names = this.names?.length
|
||
? this.names
|
||
: (await db.from('groups').where('active', true).orderBy('name').select('name')).map((g) => g.name as string).filter((n) => !isSpotGroup(n))
|
||
try {
|
||
for (const name of names) {
|
||
if (this.stop) {
|
||
await BackfillService.stop(name)
|
||
this.logger.info(`${name}: backfill stopped.`)
|
||
continue
|
||
}
|
||
const range = await BackfillService.start(name, this.days)
|
||
this.logger.info(
|
||
range
|
||
? `${name}: backfilling articles ${range.from}–${range.to} (${(range.to - range.from).toLocaleString('en')} articles, ${this.days} days).`
|
||
: `${name}: nothing to backfill.`
|
||
)
|
||
}
|
||
} finally {
|
||
await NntpService.shutdown()
|
||
}
|
||
}
|
||
}
|