Daan MeijerandClaude Opus 5.5
ee898be8c8
passwordFromSpot finds 30 more passwords in the stored descriptions (480 → 510): words between keyword and separator, trailing "!!", the value on the next line, "wachtwoord is x", "… wachtwoord. Deze is: x", "WW x" as a whole line. Listed spots were read from their overview only, so the NZB's own password and the description were unknown for 2.27M of them. SpotService.readListing reads the spot's article and NZB without importing files: details, the NZB password (over a spot-text one), the spot-text password, placeholder NZBs. worker:spots does a batch when idle; spot:postprocess works through the history (nzb_read_at). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
56 lines
2.4 KiB
TypeScript
56 lines
2.4 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import db from '@adonisjs/lucid/services/db'
|
|
import env from '#start/env'
|
|
import Shutdown from '#services/Shutdown'
|
|
import NntpService from '#services/NntpService'
|
|
import SpotService from '#services/SpotService'
|
|
|
|
const BATCH = 500
|
|
|
|
/**
|
|
* Reads the article and NZB of listed spots that were listed from their
|
|
* overview only (LOG.md 2026-10-08 (10)): description, the NZB's password or
|
|
* one in the spot's text, placeholder NZBs. Newest first; can be stopped and
|
|
* started again (worker:spots does the same when idle, a batch at a time).
|
|
* Connections: NNTP_POOL_SIZE of this process, --concurrency at a time.
|
|
*/
|
|
export default class SpotPostprocess extends BaseCommand {
|
|
public static commandName = 'spot:postprocess'
|
|
public static description = "Reads listed spots' articles and NZBs for their passwords and details."
|
|
|
|
public static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
@flags.number({ description: 'Spots read at a time (default: NNTP_POOL_SIZE)' })
|
|
declare concurrency?: number
|
|
|
|
@flags.number({ description: 'Stop after this many spots' })
|
|
declare limit?: number
|
|
|
|
public async run() {
|
|
const concurrency = this.concurrency ?? env.get('NNTP_POOL_SIZE', 4)
|
|
const [{ count }] = await db.from('spots').where('status', 'listed').whereNull('nzb_read_at').count('* as count')
|
|
this.logger.info(`${count} listed spots to read, ${concurrency} at a time.`)
|
|
const totals: Record<string, number> = {}
|
|
const started = Date.now()
|
|
let done = 0
|
|
try {
|
|
while (!Shutdown.requested && (!this.limit || done < this.limit)) {
|
|
const counts = await SpotService.readListings(Math.min(BATCH, this.limit ? this.limit - done : BATCH), concurrency)
|
|
const read = Object.values(counts).reduce((sum, n) => sum + n, 0)
|
|
if (read === 0) break
|
|
for (const [k, v] of Object.entries(counts)) totals[k] = (totals[k] ?? 0) + v
|
|
done += read
|
|
const rate = done / ((Date.now() - started) / 1000)
|
|
this.logger.info(`${done} read (${rate.toFixed(1)}/s): ${Object.entries(totals).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
|
|
// Usenet unreachable: everything was given back; don't spin on it.
|
|
if (counts.retry === read) await Shutdown.sleep(30_000)
|
|
}
|
|
} finally {
|
|
await NntpService.shutdown()
|
|
}
|
|
}
|
|
}
|