usenet-indexer/app/services/SplitService.ts

import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import IngestService from '#services/IngestService'
import NameService from '#services/NameService'
import { splitVerified } from '#lib/split/index'

/** Conflicted binaries handled per run; the rest follow on the next tick. */
const BATCH = 200

export interface SplitResult {
  parents: number
  volumes: number
  completed: number
}

/**
 * Splits conflicted obfuscated binaries (several equal-sized files under one
 * size+parts identity) into one binary per volume. Only volumes whose
 * combined part CRCs match the file CRC of their last part are created, so a
 * split can never produce a corrupt file; the rest waits for more articles.
 * See CONCEPT.md (D20).
 */
class SplitService {
  async splitPending(): Promise<SplitResult> {
    const parents = await db
      .from('binaries')
      .whereNotNull('conflicted_at')
      .whereNull('parent_binary_id')
      // split_at = 'infinity' marks binaries that cannot be split (see migration …810).
      .where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
      // Most recently active first: those are the posts still coming in.
      .orderBy('last_seen_at', 'desc')
      .limit(BATCH)

    const volumeIds: number[] = []
    for (const parent of parents) {
      volumeIds.push(...(await this.split(parent)))
    }

    // Former conflicts that turned out to be reposts (unflagged, keep their
    // split_at): no new article may arrive to trigger their completion.
    const unflagged = await db
      .from('binaries')
      .whereNull('conflicted_at')
      .whereNull('completed_at')
      .whereNull('parent_binary_id')
      .whereNotNull('split_at')
      .whereRaw("split_at <> 'infinity'")
      .limit(BATCH)
      .select('id')
    volumeIds.push(...unflagged.map((b) => Number(b.id)))

    await IngestService.matchPar2Releases(volumeIds)
    await NameService.resolve(volumeIds)
    const completed = await IngestService.claimCompleted(volumeIds)
    await IngestService.enqueue(completed)
    return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
  }

  /** Creates the newly verified volumes of one conflicted binary; returns their ids. */
  private async split(parent: any): Promise<number[]> {
    return db.transaction(async (trx) => {
      // Unverified volumes from before verification existed can never complete.
      await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()

      const { rows } = await trx.rawQuery(
        `WITH taken AS (
           SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
           WHERE v.parent_binary_id = ?
         ), candidates AS (
           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
           UNION ALL
           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
         )
         SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
        [parent.id, parent.id, parent.id]
      )
      const volumes = splitVerified(
        rows.map((r: any) => ({
          number: r.number,
          messageId: r.message_id,
          bytes: r.bytes,
          articleNumber: r.article_number === null ? null : Number(r.article_number),
          pcrc32: r.pcrc32,
          partSize: r.part_size,
          fileCrc32: r.file_crc32,
        })),
        { totalParts: parent.total_parts }
      )

      const now = new Date()
      const ids: number[] = []
      for (const articles of volumes) {
        // A verified volume is fixed, so its first article identifies it.
        const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
        const inserted = await trx
          .knexClient('binaries')
          .insert({
            file_key: fileKey,
            poster: parent.poster,
            subject: parent.subject,
            name: parent.name,
            total_parts: parent.total_parts,
            groups: JSON.stringify(parent.groups),
            posted_at: parent.posted_at,
            size: parent.size,
            release_id: parent.release_id,
            parent_binary_id: parent.id,
            first_seen_at: now,
            last_seen_at: now,
          })
          .onConflict('file_key')
          .ignore()
          .returning('id')
        if (inserted.length === 0) continue
        const id = Number(inserted[0].id)
        await trx.knexClient('parts').insert(
          articles.map((a) => ({
            binary_id: id,
            number: a.number,
            message_id: a.messageId,
            bytes: a.bytes,
            article_number: a.articleNumber,
            pcrc32: a.pcrc32,
            part_size: a.partSize,
            file_crc32: a.fileCrc32,
          }))
        )
        // The fingerprint of a volume is that of its part 1 article.
        await trx.rawQuery(
          `UPDATE binaries SET md5_16k = (
             SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
             UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
           WHERE id = ?`,
          [parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
        )
        ids.push(id)
      }

      await trx.from('binaries').where('id', parent.id).update({ split_at: now })
      return ids
    })
  }
}

export default new SplitService()