usenet-indexer/app/services/SpotService.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import indexerConfig from '#config/indexer'
import NntpService from '#services/NntpService'
import NzbImportService from '#services/NzbImportService'
import { type OverviewHeader } from '#services/IngestService'
import { parseNzb } from '#lib/nzb/index'
import {
  moderationTarget,
  spotterId,
  decodeSpotBinary,
  parseSpotArticleHeaders,
  parseSpotHeader,
  parseSpotXml,
  verifySpotHeader,
  type SpotHeader,
} from '#lib/spotnet/index'

/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
const MAX_NZB_SEGMENTS = 20
/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
/** A claimed spot whose worker died is taken again after this long. */
const RECLAIM_AFTER = '1 hour'

export function isSpotGroup(name: string): boolean {
  return indexerConfig.spotGroups.includes(name)
}

/**
 * Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
 * header holds XML with the message ids of a compressed NZB, which is
 * imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
 * (nyuu, ngPost, the skipped multi-group uploader).
 */
class SpotService {
  /** Queues the spots among a batch of one spot group's headers. Returns how many. */
  async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
    const now = new Date()
    const rows = []
    for (const [articleNumber, header] of overviews) {
      if (!header?.from || !header['message-id']) continue
      const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
      if (!spot?.title) continue
      rows.push({
        message_id: spot.messageId,
        group_name: groupName,
        article_number: Number(articleNumber),
        title: spot.title,
        tag: spot.tag || null,
        poster: spot.poster,
        category: spot.category,
        subcats: JSON.stringify(spot.subcats),
        size: spot.size || null,
        posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
        key_id: spot.keyId,
        spotter_id: spotterId(spot.selfSignedPubKey) || null,
        created_at: now,
      })
    }
    if (rows.length === 0) return 0
    await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
    return rows.length
  }

  /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
  async claim(limit: number): Promise<any[]> {
    const { rows } = await db.rawQuery(
      `UPDATE spots SET status = 'processing', processed_at = now()
       WHERE message_id IN (
         SELECT message_id FROM spots
         WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
         ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
       RETURNING *`,
      [limit]
    )
    return rows
  }

  /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
  async process(spot: any): Promise<string> {
    const update: Record<string, unknown> = { processed_at: new Date(), error: null }
    try {
      const raw = await this.fetch(`HEAD <${spot.message_id}>`)
      const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
      // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
      const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
      if (!header || !verifySpotHeader(header)) {
        update.status = 'unverified'
      } else if (moderationTarget(header)) {
        update.status = 'moderation'
        await this.moderate(header, moderationTarget(header)!)
      } else {
        const parsed = parseSpotXml(xml)
        update.title = header.title
        update.poster = header.poster
        update.spotter_id = spotterId(header.selfSignedPubKey) || null
        if (parsed?.website) update.website = parsed.website
        const segments = parsed?.nzbSegments ?? []
        if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
          update.status = 'no-nzb'
        } else {
          const bodies = []
          for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
          const nzbXml = decodeSpotBinary(bodies).toString('utf8')
          const nzb = parseNzb(nzbXml)
          if (!nzb || nzb.files.length === 0) {
            update.status = 'no-nzb'
          } else {
            const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
            const result = await NzbImportService.import(
              {
                id: null,
                filename: `${header.title}.nzb`,
                poster: header.poster,
                groups,
                date: header.stamp * 1000,
                message_ids: { '1': { id: spot.message_id } },
              },
              nzbXml,
              { name: header.title, source: 'spot', moveContainer: false }
            )
            update.status = result ? 'imported' : 'no-nzb'
            if (result) update.release_id = result.releaseId
          }
        }
      }
    } catch (error: any) {
      update.status = 'failed'
      update.error = String(error.message).slice(0, 500)
      logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
    }
    await db.from('spots').where('message_id', spot.message_id).update(update)
    return update.status as string
  }

  /**
   * Applies a delete/dispose/remove: the target spot is marked `removed`
   * (its release is then hidden from search). As in Spotweb, a personal
   * dispose must come from the same spotter within 5 days; a moderator's
   * (signed with Spotnet's key, no personal key) always counts.
   */
  private async moderate(header: SpotHeader, targetId: string) {
    const target = await db.from('spots').where('message_id', targetId).first()
    if (!target) return
    const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
    if (personal) {
      const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
    }
    await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
  }

  /** One HEAD or BODY, as raw bytes. */
  private async fetch(command: string): Promise<Buffer> {
    const conn = await NntpService.acquire()
    let failed = false
    try {
      const { data } = await conn._artcmd(command)
      return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
    } catch (error: any) {
      // 430 (no such article) is an answer, not a broken connection.
      failed = !/^4\d\d/.test(String(error.message))
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
  }
}

export default new SpotService()