import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import indexerConfig from '#config/indexer' import NntpService from '#services/NntpService' import NzbImportService from '#services/NzbImportService' import { type OverviewHeader } from '#services/IngestService' import { parseNzb } from '#lib/nzb/index' import { moderationTarget, spotterId, decodeSpotBinary, parseSpotArticleHeaders, parseSpotHeader, parseSpotXml, verifySpotHeader, type SpotHeader, } from '#lib/spotnet/index' /** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */ const MAX_NZB_SEGMENTS = 20 /** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */ const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60 /** A claimed spot whose worker died is taken again after this long. */ const RECLAIM_AFTER = '1 hour' export function isSpotGroup(name: string): boolean { return indexerConfig.spotGroups.includes(name) } /** * Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full * header holds XML with the message ids of a compressed NZB, which is * imported like a posted NZB (D26). Spot NZBs name many obfuscated posts * (nyuu, ngPost, the skipped multi-group uploader). */ class SpotService { /** Queues the spots among a batch of one spot group's headers. Returns how many. */ async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise { const now = new Date() const rows = [] for (const [articleNumber, header] of overviews) { if (!header?.from || !header['message-id']) continue const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date) if (!spot?.title) continue rows.push({ message_id: spot.messageId, group_name: groupName, article_number: Number(articleNumber), title: spot.title, tag: spot.tag || null, poster: spot.poster, category: spot.category, subcats: JSON.stringify(spot.subcats), size: spot.size || null, posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null, key_id: spot.keyId, spotter_id: spotterId(spot.selfSignedPubKey) || null, created_at: now, }) } if (rows.length === 0) return 0 await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore() return rows.length } /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */ async claim(limit: number): Promise { const { rows } = await db.rawQuery( `UPDATE spots SET status = 'processing', processed_at = now() WHERE message_id IN ( SELECT message_id FROM spots WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}') ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED) RETURNING *`, [limit] ) return rows } /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */ async process(spot: any): Promise { const update: Record = { processed_at: new Date(), error: null } try { const raw = await this.fetch(`HEAD <${spot.message_id}>`) const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1')) // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled. const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date) if (!header || !verifySpotHeader(header)) { update.status = 'unverified' } else if (moderationTarget(header)) { update.status = 'moderation' await this.moderate(header, moderationTarget(header)!) } else { const parsed = parseSpotXml(xml) update.title = header.title update.poster = header.poster update.spotter_id = spotterId(header.selfSignedPubKey) || null if (parsed?.website) update.website = parsed.website const segments = parsed?.nzbSegments ?? [] if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) { update.status = 'no-nzb' } else { const bodies = [] for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`)) const nzbXml = decodeSpotBinary(bodies).toString('utf8') const nzb = parseNzb(nzbXml) if (!nzb || nzb.files.length === 0) { update.status = 'no-nzb' } else { const groups = [...new Set(nzb.files.flatMap((f) => f.groups))] const result = await NzbImportService.import( { id: null, filename: `${header.title}.nzb`, poster: header.poster, groups, date: header.stamp * 1000, message_ids: { '1': { id: spot.message_id } }, }, nzbXml, { name: header.title, source: 'spot', moveContainer: false } ) update.status = result ? 'imported' : 'no-nzb' if (result) update.release_id = result.releaseId } } } } catch (error: any) { update.status = 'failed' update.error = String(error.message).slice(0, 500) logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`) } await db.from('spots').where('message_id', spot.message_id).update(update) return update.status as string } /** * Applies a delete/dispose/remove: the target spot is marked `removed` * (its release is then hidden from search). As in Spotweb, a personal * dispose must come from the same spotter within 5 days; a moderator's * (signed with Spotnet's key, no personal key) always counts. */ private async moderate(header: SpotHeader, targetId: string) { const target = await db.from('spots').where('message_id', targetId).first() if (!target) return const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : '' if (personal) { const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0) if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return } await db.from('spots').where('message_id', targetId).update({ status: 'removed' }) logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`) } /** One HEAD or BODY, as raw bytes. */ private async fetch(command: string): Promise { const conn = await NntpService.acquire() let failed = false try { const { data } = await conn._artcmd(command) return Buffer.from(data.buffer, data.byteOffset, data.byteLength) } catch (error: any) { // 430 (no such article) is an answer, not a broken connection. failed = !/^4\d\d/.test(String(error.message)) throw error } finally { NntpService.release(conn, failed) } } } export default new SpotService()