usenet-indexer/app/services/SeedService.ts

import db from '@adonisjs/lucid/services/db'
import type { AggregatedFile } from '#lib/aggregate/index'
import IngestService, { type IngestSegment } from '#services/IngestService'
import FileStore from '#services/FileStore'
import SfvService from '#services/SfvService'
import ContentService from '#services/ContentService'
import QueueService from '#services/QueueService'

export interface SeedStoreResult {
  files: number
  /** Files that existed already (same file_key, e.g. from live indexing or a crosspost). */
  existing: number
  releases: number
  inspections: number
}

/**
 * Stores files an edge node aggregated from headers (CONCEPT.md D37): their
 * releases, the files with their segment lists, and one completed binary per
 * file, as worker:file would have left it, so par2 naming, inspection and GC
 * treat them like any other file. A batch is a few statements, not one per
 * article.
 */
class SeedService {
  async store(files: AggregatedFile<IngestSegment>[]): Promise<SeedStoreResult> {
    const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0 }
    if (files.length === 0) return result
    const knex = db.connection().getWriteClient()
    const now = new Date()

    // A file's identity and release come from its first segment, its date from the earliest.
    const firstOf = (file: AggregatedFile<IngestSegment>) => file.segments[0]
    const postedOf = (file: AggregatedFile<IngestSegment>) => {
      const dates = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t))
      return dates.length > 0 ? new Date(Math.min(...dates)) : undefined
    }
    const releaseIds = await IngestService.upsertReleases(
      knex,
      files.map((file) => ({ ...firstOf(file), postedAt: postedOf(file) })),
      now
    )
    result.releases = releaseIds.size

    const rows = files.map((file) => {
      const first = firstOf(file)
      const postedAt = postedOf(file)
      return {
        file_key: file.fileKey,
        filename: first.name,
        poster: first.poster,
        date: (postedAt ?? now).getTime(),
        parts: file.segments.length,
        segments: file.segments.map((s) => ({ number: s.number, id: s.messageId, size: s.bytes })),
        groups: JSON.stringify(first.groups),
        release_id: first.release ? (releaseIds.get(first.release.key) ?? null) : null,
        size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
        created_at: now,
        updated_at: now,
      }
    })
    const inserted = await FileStore.insert(rows)
    result.files = inserted.length
    result.existing = rows.length - inserted.length

    const fileIds = new Map<string, number>()
    for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) {
      fileIds.set(row.file_key, Number(row.id))
    }
    // The completed binary worker:file would have left behind; an existing one just learns its file.
    const binaries = rows.map((row, i) => ({
      file_key: row.file_key,
      poster: row.poster,
      subject: firstOf(files[i]).subject,
      name: row.filename,
      total_parts: firstOf(files[i]).totalParts,
      groups: row.groups,
      posted_at: postedOf(files[i]) ?? null,
      first_seen_at: now,
      last_seen_at: now,
      completed_at: now,
      release_id: row.release_id,
      file_id: fileIds.get(row.file_key) ?? null,
    }))
    binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
    await knex('binaries')
      .insert(binaries)
      .onConflict('file_key')
      .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any })

    // As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
    const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row]))
    const added = inserted.flatMap((file) => {
      const row = rowById.get(file.id)
      return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
    })
    await SfvService.claimAll(added)
    const candidates = added
      .filter((file) => ContentService.isCandidate(file.name))
      .map((file) => ({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }))
    if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates)
    result.inspections = candidates.length
    return result
  }
}

export default new SeedService()