usenet-indexer/app/services/spotter.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import { SOURCES } from '#lib/release/source'
import env from '#start/env'
import coordinatorConfig from '#config/coordinator'
import { MissingArticle, XoverClient } from '#lib/nntp/index'
import {
  decodeSpotText,
  moderationTarget,
  parseSpotArticleHeaders,
  parseSpotHeader,
  parseSpotXml,
  passwordFromSpot,
  spotterId,
  verifySpotHeader,
} from '#lib/spotnet/index'

/** Overviews asked for at a time. */
const XOVER_STEP = 1000
/** A personal dispose counts within this long of the spot (Spotweb). */
const DISPOSE_WINDOW_SECONDS = 5 * 86400

export interface SpotterRound {
  read: number
  listed: number
  unverified: number
  moderation: number
}

/**
 * The spotter (REBUILD.md, spots): Spotnet's spots, from the headers of
 * free.pt into `spots`, on one NNTP connection of its own. Every few
 * minutes it reads the overviews after the last article it saw (the first
 * time the last SPOT_BACKFILL), and for each spot the full header: its
 * signature is checked there (lib/spotnet, as Spotweb does), and the X-XML
 * gives the description, the website and the NZB's articles. A password the
 * title (`{{pw}}`) or the description names is kept with the spot; the spot
 * inspector carries it to the releases. A delete/dispose message marks its
 * target `removed` and takes back what the target gave the releases.
 */
export class Spotter {
  private stopped = false
  private client?: XoverClient

  stop() {
    this.stopped = true
    this.client?.close()
  }

  async run() {
    while (!this.stopped) {
      const started = Date.now()
      try {
        const round = await this.round()
        if (round.read > 0) {
          logger.info(`Spots: ${round.read} articles read, ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
        }
      } catch (error: any) {
        logger.error(`Spot round failed: ${error.message}`)
        this.client?.close()
        this.client = undefined
      }
      const wait = coordinatorConfig.spots.everySeconds * 1000
      // Quiet until the next round: closed (nntpIdleCloseMs), opened again then.
      const idleMs = coordinatorConfig.nntpIdleCloseMs
      if (idleMs > 0 && wait - (Date.now() - started) >= idleMs) {
        this.client?.close()
        this.client = undefined
      }
      for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
    }
  }

  private async connect(): Promise<XoverClient> {
    this.client ??= await XoverClient.connect({
      host: env.get('NNTP_HOST')!,
      port: env.get('NNTP_PORT', 563),
      user: env.get('NNTP_USER'),
      password: env.get('NNTP_PASSWORD'),
      secure: env.get('NNTP_SECURE', true),
      compress: false,
      timeoutMs: coordinatorConfig.nntpTimeoutMs,
    })
    return this.client
  }

  /** The articles after the last one seen, up to the group's high. */
  async round(): Promise<SpotterRound> {
    const { group, backfill } = coordinatorConfig.spots
    const client = await this.connect()
    const { low, high } = await client.group(group)
    const state = await db.from('spot_state').where('group_name', group).first()
    let next = state ? Number(state.last_article) + 1 : Math.max(low, high - backfill + 1)
    const round: SpotterRound = { read: 0, listed: 0, unverified: 0, moderation: 0 }
    while (next <= high && !this.stopped) {
      const to = Math.min(high, next + XOVER_STEP - 1)
      const overviews = await client.xover(next, to)
      round.read += overviews.length
      for (const [number, fields] of overviews) {
        if (this.stopped) return round
        const messageId = (fields['message-id'] ?? '').replace(/^<|>$/g, '')
        if (!messageId || !parseSpotHeader(fields.from ?? '', fields.subject ?? '', messageId, fields.date)) continue
        const outcome = await this.read(client, messageId, number)
        if (outcome) round[outcome]++
      }
      await db
        .table('spot_state')
        .insert({ group_name: group, last_article: to })
        .onConflict('group_name')
        .merge({ last_article: to })
      next = to + 1
    }
    return round
  }

  /** One spot by its full header; what it was, or null when it was there already or is gone. */
  async read(client: XoverClient, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
    if (await db.from('spots').where('message_id', messageId).first()) return null
    let raw: Buffer
    try {
      raw = await client.head(messageId)
    } catch (error) {
      if (error instanceof MissingArticle) return null
      throw error
    }
    return this.store(raw.toString('latin1'), messageId, articleNumber)
  }

  /** Verifies and stores a spot from its raw header (the tests feed this directly). */
  async store(raw: string, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
    const { headers, xml: xmlText } = parseSpotArticleHeaders(raw)
    const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
    if (!header) return null
    const verified = verifySpotHeader(header)
    const xml = verified ? parseSpotXml(decodeSpotText(xmlText)) : null
    const target = verified ? moderationTarget(header) : null
    const title = xml?.title || header.title
    // Kept apart by where it came from: the title's {{pw}}, else the description's "Wachtwoord: …".
    const fromTitle = verified && !target ? passwordFromSpot(title, null) : null
    const fromText = verified && !target && !fromTitle ? passwordFromSpot('', xml?.description) : null
    const status = !verified ? 'unverified' : target ? 'moderation' : 'listed'
    await db
      .table('spots')
      .insert({
        message_id: messageId,
        article_number: articleNumber,
        title: title.slice(0, 2000),
        tag: header.tag || null,
        poster: header.poster,
        category: header.category,
        subcats: JSON.stringify(header.subcats),
        size: header.size || null,
        posted_at: header.stamp ? new Date(header.stamp * 1000) : null,
        key_id: header.keyId,
        spotter_id: header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : null,
        description: xml?.description || null,
        website: xml?.website || null,
        nzb_segments: xml?.nzbSegments.length ? xml.nzbSegments : null,
        password: fromTitle ?? fromText,
        password_source: fromTitle ? 'spot-title' : fromText ? 'spot-text' : null,
        status,
      })
      .onConflict('message_id')
      .ignore()
    if (target) await this.moderate(header.selfSignedPubKey, header.stamp, target)
    return status
  }

  /**
   * A delete/dispose/remove: the target is `removed`, and its links and passwords go from the releases. As in
   * Spotweb, a personal dispose (the poster's own key) counts only from the same spotter within 5 days; a
   * moderator's (Spotnet's key) always.
   */
  private async moderate(pubKey: string, stamp: number, targetId: string) {
    const target = await db.from('spots').where('message_id', targetId).first()
    if (!target) return
    const personal = pubKey.length > 50 ? spotterId(pubKey) : ''
    if (personal) {
      const age = stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
    }
    await db.transaction(async (trx) => {
      await trx.from('spots').where('message_id', targetId).update({ status: 'removed' })
      await trx.from('release_spots').where('spot_id', targetId).delete()
      await trx.from('release_passwords').where('spot_id', targetId).delete()
      // A release the spot made of its own goes with it.
      if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', SOURCES.spot).delete()
    })
    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
  }
}