Spots: more password phrasings; read listed spots' articles and NZBs
Test and deploy / test (push) Successful in 57s
Test and deploy / deploy (push) Successful in 4m18s

passwordFromSpot finds 30 more passwords in the stored descriptions (480
→ 510): words between keyword and separator, trailing "!!", the value on
the next line, "wachtwoord is x", "… wachtwoord. Deze is: x", "WW x" as a
whole line.

Listed spots were read from their overview only, so the NZB's own
password and the description were unknown for 2.27M of them.
SpotService.readListing reads the spot's article and NZB without
importing files: details, the NZB password (over a spot-text one), the
spot-text password, placeholder NZBs. worker:spots does a batch when
idle; spot:postprocess works through the history (nzb_read_at).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-08 13:46:32 +02:00
1 parent 36f84ae3e6
commit ee898be8c8
8 files changed
+322 -17

No files matched your search

+94 -4
View File
@@ -297,6 +297,90 @@ class SpotService {
return read
}
/**
* Takes up to `limit` listed spots whose article and NZB haven't been read,
* newest first, marking them so another reader skips them.
*/
async claimListings(limit: number): Promise<any[]> {
const { rows } = await db.rawQuery(
`UPDATE spots SET nzb_read_at = now()
WHERE message_id IN (
SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL
ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED)
RETURNING *`,
[limit]
)
return rows
}
/**
* Reads a listed spot's article and NZB without importing its files
* (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's
* own password (`password_source = 'nzb'`, over one from the spot's text),
* else one in the spot's text; a placeholder NZB hides the spot (`dummy`).
* Returns what it found. A connection failure gives the spot back for later.
*/
async readListing(spot: any): Promise<string> {
try {
const head = await this.readHead(spot.message_id)
if (!head) {
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
return 'unverified'
}
const update: Record<string, unknown> = this.details(head.xml)
let outcome = 'nothing'
let nzbXml: string | null = null
try {
nzbXml = await this.readNzb(head)
} catch (error: any) {
if (!/^4\d\d/.test(String(error.message))) throw error
// The NZB's articles are gone: the listing can't be downloaded any more.
update.error = 'NZB articles gone'
outcome = 'nzb-gone'
}
const nzb = nzbXml ? parseNzb(nzbXml) : null
if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) {
update.status = 'dummy'
outcome = 'dummy'
} else if (nzb?.meta.password && spot.release_id) {
await db
.from('releases')
.where('id', spot.release_id)
.where((q) => q.whereNull('password').orWhere('password_source', 'spot'))
.update({ password: nzb.meta.password, password_source: 'nzb' })
outcome = 'nzb-password'
}
await db.from('spots').where('message_id', spot.message_id).update(update)
if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password'
return outcome
} catch (error: any) {
// The spot's own article is gone (430): nothing to read later either.
if (/^4\d\d/.test(String(error.message))) {
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
return 'gone'
}
await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null })
logger.warn(`Spot ${spot.message_id} listing: ${error.message}`)
return 'retry'
}
}
/** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */
async readListings(limit: number, concurrency: number): Promise<Record<string, number>> {
const spots = await this.claimListings(limit)
const counts: Record<string, number> = {}
let next = 0
await Promise.all(
Array.from({ length: Math.max(1, concurrency) }, async () => {
while (next < spots.length) {
const outcome = await this.readListing(spots[next++])
counts[outcome] = (counts[outcome] ?? 0) + 1
}
})
)
return counts
}
/** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
async image(spot: any): Promise<Buffer | null> {
const segments: string[] = spot.image?.segments ?? []
@@ -315,13 +399,19 @@ class SpotService {
return { header, xml: parseSpotXml(decodeSpotText(xml)) }
}
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
/** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */
private async readNzb(head: { xml: SpotXml | null }): Promise<string | null> {
const segments = head.xml?.nzbSegments ?? []
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null
const bodies = []
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
return decodeSpotBinary(bodies).toString('utf8')
}
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
const nzbXml = await this.readNzb(head)
if (!nzbXml) return { status: 'no-nzb' }
const nzb = parseNzb(nzbXml)
if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
const { header } = head