usenet-indexer/app/services/BinaryFileService.ts

import db from '@adonisjs/lucid/services/db'
import FileStore from '#services/FileStore'
import SfvService from '#services/SfvService'
import ContentService from '#services/ContentService'
import InspectionQueue from '#services/InspectionQueue'

export interface BinaryFilesResult {
  /** New files. */
  saved: number
  /** Binaries whose file already existed (same first segment or file_key). */
  existing: number
  /** Conflicted, gone or already linked: nothing to do. */
  skipped: number
  /** Not all parts stored yet (the job is retried for these). */
  incomplete: number[]
  inspections: number
}

/**
 * Turns completed binaries into files (worker:file), a batch at a time
 * (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and
 * files that exist already, one INSERT for the new files (so the files
 * statement trigger updates each release once), and one UPDATE and DELETE
 * for the binaries' links and copied parts. It was one job of ~8 queries per
 * binary, ~700 a minute.
 */
class BinaryFileService {
  async store(binaryIds: number[]): Promise<BinaryFilesResult> {
    const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 }
    const ids = [...new Set(binaryIds)]
    const binaries = await db
      .from('binaries')
      .whereIn('id', ids)
      .whereNull('conflicted_at')
      .whereNull('file_id')
      .select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts')
    result.skipped = ids.length - binaries.length
    if (binaries.length === 0) return result

    const partsOf = new Map<number, { number: number; id: string; size: number }[]>()
    const parts = await db
      .from('parts')
      .whereIn('binary_id', binaries.map((b) => b.id))
      .orderBy([{ column: 'binary_id' }, { column: 'number' }])
      .select('binary_id', 'number', 'message_id', 'bytes')
    for (const part of parts) {
      const list = partsOf.get(Number(part.binary_id)) ?? []
      list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) })
      partsOf.set(Number(part.binary_id), list)
    }
    const complete = binaries.filter((b) => {
      const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts
      if (!ok) result.incomplete.push(Number(b.id))
      return ok
    })
    if (complete.length === 0) return result

    // Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list).
    const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id)
    const byFirst = new Map<string, number>()
    for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) {
      byFirst.set(row.first_message_id, Number(row.id))
    }
    const fileIdOf = new Map<number, number>()
    const fresh = complete.filter((b) => {
      const existing = byFirst.get(partsOf.get(Number(b.id))![0].id)
      if (existing) fileIdOf.set(Number(b.id), existing)
      return !existing
    })

    const now = new Date()
    // file_key is unique: a second completion of the same upload inserts nothing.
    const inserted = await FileStore.insert(
      fresh.map((b) => {
        const segments = partsOf.get(Number(b.id))!
        return {
          file_key: b.file_key,
          filename: b.name,
          poster: b.poster,
          date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(),
          parts: segments.length,
          segments,
          groups: JSON.stringify(b.groups),
          release_id: b.release_id,
          real_name: b.real_name,
          size: segments.reduce((sum, s) => sum + s.size, 0),
          created_at: now,
          updated_at: now,
        }
      })
    )
    const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id]))
    const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key)
    const byKey = new Map<string, number>()
    if (missing.length > 0) {
      for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id))
    }
    for (const b of fresh) {
      const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
      if (id) fileIdOf.set(Number(b.id), id)
    }
    const fresher = new Map(fresh.map((b) => [b.file_key, b]))
    result.saved = inserted.length
    result.existing = fileIdOf.size - inserted.length

    // Links, locked in file_key order like ingest's upsert, so the two can't deadlock.
    const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key))
    if (linked.length > 0) {
      await db.transaction(async (trx) => {
        await trx.rawQuery(
          `SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
          [linked.map((b) => b.id)]
        )
        // Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
        const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
        await trx.rawQuery(
          `UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
           FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
          [linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
        )
      })
      // Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
      await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)])
    }

    // As for one file before: an SFV posted earlier may list them (D34), then inspection.
    const added = inserted.flatMap((row) => {
      const b = row.file_key ? fresher.get(row.file_key) : undefined
      return b
        ? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }]
        : []
    })
    if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file))
    // By the posted name, as before: inspection skips later volumes and readable media names anyway.
    const jobs = added
      .filter((file) => ContentService.isCandidate(file.posted))
      .map((file) => file.id)
    if (jobs.length > 0) await InspectionQueue.add(jobs)
    result.inspections = jobs.length
    return result
  }
}

export default new BinaryFileService()