Spots: more password phrasings; read listed spots' articles and NZBs
passwordFromSpot finds 30 more passwords in the stored descriptions (480 → 510): words between keyword and separator, trailing "!!", the value on the next line, "wachtwoord is x", "… wachtwoord. Deze is: x", "WW x" as a whole line. Listed spots were read from their overview only, so the NZB's own password and the description were unknown for 2.27M of them. SpotService.readListing reads the spot's article and NZB without importing files: details, the NZB password (over a spot-text one), the spot-text password, placeholder NZBs. worker:spots does a batch when idle; spot:postprocess works through the history (nzb_read_at). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
36f84ae3e6
commit
ee898be8c8
8 files changed
+322
-17
No files matched your search
@@ -297,6 +297,90 @@ class SpotService {
|
||||
return read
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes up to `limit` listed spots whose article and NZB haven't been read,
|
||||
* newest first, marking them so another reader skips them.
|
||||
*/
|
||||
async claimListings(limit: number): Promise<any[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE spots SET nzb_read_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL
|
||||
ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a listed spot's article and NZB without importing its files
|
||||
* (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's
|
||||
* own password (`password_source = 'nzb'`, over one from the spot's text),
|
||||
* else one in the spot's text; a placeholder NZB hides the spot (`dummy`).
|
||||
* Returns what it found. A connection failure gives the spot back for later.
|
||||
*/
|
||||
async readListing(spot: any): Promise<string> {
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
return 'unverified'
|
||||
}
|
||||
const update: Record<string, unknown> = this.details(head.xml)
|
||||
let outcome = 'nothing'
|
||||
let nzbXml: string | null = null
|
||||
try {
|
||||
nzbXml = await this.readNzb(head)
|
||||
} catch (error: any) {
|
||||
if (!/^4\d\d/.test(String(error.message))) throw error
|
||||
// The NZB's articles are gone: the listing can't be downloaded any more.
|
||||
update.error = 'NZB articles gone'
|
||||
outcome = 'nzb-gone'
|
||||
}
|
||||
const nzb = nzbXml ? parseNzb(nzbXml) : null
|
||||
if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) {
|
||||
update.status = 'dummy'
|
||||
outcome = 'dummy'
|
||||
} else if (nzb?.meta.password && spot.release_id) {
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.where((q) => q.whereNull('password').orWhere('password_source', 'spot'))
|
||||
.update({ password: nzb.meta.password, password_source: 'nzb' })
|
||||
outcome = 'nzb-password'
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password'
|
||||
return outcome
|
||||
} catch (error: any) {
|
||||
// The spot's own article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
return 'gone'
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null })
|
||||
logger.warn(`Spot ${spot.message_id} listing: ${error.message}`)
|
||||
return 'retry'
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */
|
||||
async readListings(limit: number, concurrency: number): Promise<Record<string, number>> {
|
||||
const spots = await this.claimListings(limit)
|
||||
const counts: Record<string, number> = {}
|
||||
let next = 0
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.max(1, concurrency) }, async () => {
|
||||
while (next < spots.length) {
|
||||
const outcome = await this.readListing(spots[next++])
|
||||
counts[outcome] = (counts[outcome] ?? 0) + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
return counts
|
||||
}
|
||||
|
||||
/** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
|
||||
async image(spot: any): Promise<Buffer | null> {
|
||||
const segments: string[] = spot.image?.segments ?? []
|
||||
@@ -315,13 +399,19 @@ class SpotService {
|
||||
return { header, xml: parseSpotXml(decodeSpotText(xml)) }
|
||||
}
|
||||
|
||||
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
|
||||
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
|
||||
/** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */
|
||||
private async readNzb(head: { xml: SpotXml | null }): Promise<string | null> {
|
||||
const segments = head.xml?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
|
||||
return decodeSpotBinary(bodies).toString('utf8')
|
||||
}
|
||||
|
||||
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
|
||||
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
|
||||
const nzbXml = await this.readNzb(head)
|
||||
if (!nzbXml) return { status: 'no-nzb' }
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
|
||||
const { header } = head
|
||||
|
||||
Reference in new issue
Block a user