implemented a lot of features

app/services/SpotService.ts

@@ -0,0 +1,172 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import indexerConfig from '#config/indexer'
+import NntpService from '#services/NntpService'
+import NzbImportService from '#services/NzbImportService'
+import { type OverviewHeader } from '#services/IngestService'
+import { parseNzb } from '#lib/nzb/index'
+import {
+  moderationTarget,
+  spotterId,
+  decodeSpotBinary,
+  parseSpotArticleHeaders,
+  parseSpotHeader,
+  parseSpotXml,
+  verifySpotHeader,
+  type SpotHeader,
+} from '#lib/spotnet/index'
+
+/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
+const MAX_NZB_SEGMENTS = 20
+/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
+const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
+/** A claimed spot whose worker died is taken again after this long. */
+const RECLAIM_AFTER = '1 hour'
+
+export function isSpotGroup(name: string): boolean {
+  return indexerConfig.spotGroups.includes(name)
+}
+
+/**
+ * Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
+ * header holds XML with the message ids of a compressed NZB, which is
+ * imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
+ * (nyuu, ngPost, the skipped multi-group uploader).
+ */
+class SpotService {
+  /** Queues the spots among a batch of one spot group's headers. Returns how many. */
+  async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
+    const now = new Date()
+    const rows = []
+    for (const [articleNumber, header] of overviews) {
+      if (!header?.from || !header['message-id']) continue
+      const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
+      if (!spot?.title) continue
+      rows.push({
+        message_id: spot.messageId,
+        group_name: groupName,
+        article_number: Number(articleNumber),
+        title: spot.title,
+        tag: spot.tag || null,
+        poster: spot.poster,
+        category: spot.category,
+        subcats: JSON.stringify(spot.subcats),
+        size: spot.size || null,
+        posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
+        key_id: spot.keyId,
+        spotter_id: spotterId(spot.selfSignedPubKey) || null,
+        created_at: now,
+      })
+    }
+    if (rows.length === 0) return 0
+    await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
+    return rows.length
+  }
+
+  /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
+  async claim(limit: number): Promise<any[]> {
+    const { rows } = await db.rawQuery(
+      `UPDATE spots SET status = 'processing', processed_at = now()
+       WHERE message_id IN (
+         SELECT message_id FROM spots
+         WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
+         ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
+       RETURNING *`,
+      [limit]
+    )
+    return rows
+  }
+
+  /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
+  async process(spot: any): Promise<string> {
+    const update: Record<string, unknown> = { processed_at: new Date(), error: null }
+    try {
+      const raw = await this.fetch(`HEAD <${spot.message_id}>`)
+      const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
+      // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
+      const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
+      if (!header || !verifySpotHeader(header)) {
+        update.status = 'unverified'
+      } else if (moderationTarget(header)) {
+        update.status = 'moderation'
+        await this.moderate(header, moderationTarget(header)!)
+      } else {
+        const parsed = parseSpotXml(xml)
+        update.title = header.title
+        update.poster = header.poster
+        update.spotter_id = spotterId(header.selfSignedPubKey) || null
+        if (parsed?.website) update.website = parsed.website
+        const segments = parsed?.nzbSegments ?? []
+        if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
+          update.status = 'no-nzb'
+        } else {
+          const bodies = []
+          for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
+          const nzbXml = decodeSpotBinary(bodies).toString('utf8')
+          const nzb = parseNzb(nzbXml)
+          if (!nzb || nzb.files.length === 0) {
+            update.status = 'no-nzb'
+          } else {
+            const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
+            const result = await NzbImportService.import(
+              {
+                id: null,
+                filename: `${header.title}.nzb`,
+                poster: header.poster,
+                groups,
+                date: header.stamp * 1000,
+                message_ids: { '1': { id: spot.message_id } },
+              },
+              nzbXml,
+              { name: header.title, source: 'spot', moveContainer: false }
+            )
+            update.status = result ? 'imported' : 'no-nzb'
+            if (result) update.release_id = result.releaseId
+          }
+        }
+      }
+    } catch (error: any) {
+      update.status = 'failed'
+      update.error = String(error.message).slice(0, 500)
+      logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
+    }
+    await db.from('spots').where('message_id', spot.message_id).update(update)
+    return update.status as string
+  }
+
+  /**
+   * Applies a delete/dispose/remove: the target spot is marked `removed`
+   * (its release is then hidden from search). As in Spotweb, a personal
+   * dispose must come from the same spotter within 5 days; a moderator's
+   * (signed with Spotnet's key, no personal key) always counts.
+   */
+  private async moderate(header: SpotHeader, targetId: string) {
+    const target = await db.from('spots').where('message_id', targetId).first()
+    if (!target) return
+    const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
+    if (personal) {
+      const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
+      if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
+    }
+    await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
+    logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
+  }
+
+  /** One HEAD or BODY, as raw bytes. */
+  private async fetch(command: string): Promise<Buffer> {
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      const { data } = await conn._artcmd(command)
+      return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
+    } catch (error: any) {
+      // 430 (no such article) is an answer, not a broken connection.
+      failed = !/^4\d\d/.test(String(error.message))
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+  }
+}
+
+export default new SpotService()