Daan MeijerandClaude Opus 5.5
ee898be8c8
passwordFromSpot finds 30 more passwords in the stored descriptions (480 → 510): words between keyword and separator, trailing "!!", the value on the next line, "wachtwoord is x", "… wachtwoord. Deze is: x", "WW x" as a whole line. Listed spots were read from their overview only, so the NZB's own password and the description were unknown for 2.27M of them. SpotService.readListing reads the spot's article and NZB without importing files: details, the NZB password (over a spot-text one), the spot-text password, placeholder NZBs. worker:spots does a batch when idle; spot:postprocess works through the history (nzb_read_at). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
3.3 KiB
TypeScript
77 lines
3.3 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import env from '#start/env'
|
|
import Shutdown from '#services/Shutdown'
|
|
import NntpService from '#services/NntpService'
|
|
import SpotService from '#services/SpotService'
|
|
|
|
const IDLE_MS = 30_000
|
|
/** Spots claimed at a time; most of a history backfill is listed in bulk. */
|
|
const BATCH = 500
|
|
/** Imported spots whose details are read per idle round. */
|
|
const DETAILS_BATCH = 50
|
|
/** Listed spots whose article and NZB are read per idle round (passwords, dummies). */
|
|
const LISTINGS_BATCH = 50
|
|
|
|
/**
|
|
* Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
|
|
* downloads its compressed NZB (a few KB to a few MB) and imports it. When
|
|
* idle, reads what listed spots were listed without: details, passwords.
|
|
*/
|
|
export default class SpotsWorker extends BaseCommand {
|
|
public static commandName = 'worker:spots'
|
|
public static description = 'Reads Spotnet spots: verifies them and imports their NZBs.'
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
@flags.number({ description: 'Stop after this many spots (default: keep running)' })
|
|
declare limit?: number
|
|
|
|
public async run() {
|
|
const concurrency = env.get('SPOT_CONCURRENCY', 4)
|
|
this.logger.info(`Spot worker started (concurrency ${concurrency}).`)
|
|
const counts: Record<string, number> = {}
|
|
let done = 0
|
|
try {
|
|
while (!Shutdown.requested && (!this.limit || done < this.limit)) {
|
|
try {
|
|
const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
|
|
if (batch.length === 0) {
|
|
if (this.limit) break
|
|
// Idle: read the details of imported spots that don't have them yet (newest first), then
|
|
// the article and NZB of listed spots (their passwords; LOG.md 2026-10-08 (10)).
|
|
if ((await SpotService.fillDetails(DETAILS_BATCH)) > 0) continue
|
|
const listings = await SpotService.readListings(LISTINGS_BATCH, concurrency)
|
|
const read = Object.values(listings).reduce((sum, n) => sum + n, 0)
|
|
if (read === 0 || listings.retry === read) await Shutdown.sleep(IDLE_MS)
|
|
continue
|
|
}
|
|
// Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
|
|
const listable = batch.filter((spot) => SpotService.isListable(spot))
|
|
const single = batch.filter((spot) => !SpotService.isListable(spot))
|
|
counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
|
|
let next = 0
|
|
await Promise.all(
|
|
Array.from({ length: concurrency }, async () => {
|
|
while (next < single.length) {
|
|
const status = await SpotService.process(single[next++])
|
|
counts[status] = (counts[status] ?? 0) + 1
|
|
}
|
|
})
|
|
)
|
|
done += batch.length
|
|
this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
|
|
} catch (err: any) {
|
|
// E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
|
|
this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
|
|
await Shutdown.sleep(30_000)
|
|
}
|
|
}
|
|
} finally {
|
|
await NntpService.shutdown()
|
|
}
|
|
}
|
|
}
|