usenet-indexer/app/services/Par2Service.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import NntpService from '#services/NntpService'
import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
import { decodeSegment, YencFile } from '#lib/yenc/index'
import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'

/** Par2 index files are small; recovery volumes can be large and are not needed. */
const MAX_PARTS = 20
/** Size-only matching below this is too ambiguous. */
const SIZE_ONLY_MIN = 1024 * 1024

function countBy<T>(values: T[]): Map<T, number> {
  const counts = new Map<T, number>()
  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
  return counts
}

export interface Par2Result {
  releaseId: number
  releaseName: string
  listed: number
  matched: number
}

/**
 * Recognises releases of obfuscated posts from their par2 files: the par2
 * lists the real name and exact size of every file in the set, and
 * body-derived binaries carry their exact size. See CONCEPT.md (D16).
 */
class Par2Service {
  /**
   * Reads a par2 file whose first segment is already decoded, downloading the
   * remaining parts if there are any. Returns null when it is not par2.
   */
  async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
    if (!isPar2(first)) return null
    const ids = Object.entries(messageIds)
      .sort(([a], [b]) => Number(a) - Number(b))
      .map(([, part]) => part.id)
    if (ids.length > MAX_PARTS) return null
    if (ids.length === 1) return parsePar2(first)

    let assembled: YencFile | undefined
    const conn = await NntpService.acquire()
    let failed = false
    try {
      for (const id of ids) {
        const segment = decodeSegment(await NntpService.fetchBody(conn, id))
        assembled ??= YencFile.fromSegment(segment)
        assembled.add(segment)
      }
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
    return parsePar2(assembled!.getBuffer())
  }

  async attach(binary: any, set: Par2Set): Promise<Par2Result> {
    const result = await db.transaction(async (trx) => {
      // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
      // the set into a release the files already belong to; size alone is
      // ambiguous across posts (standard volume sizes recur).
      const exact = await trx
        .from('binaries')
        .whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
        .select('id', 'release_id')

      // Size only, for files without a fingerprint: the size must be listed
      // once in this set, be large, and belong to exactly one binary nearby.
      const listedOnce = [...countBy(set.files.map((f) => f.length))]
        .filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
        .map(([size]) => size)
      const bySize = listedOnce.length
        ? await trx
            .from('binaries')
            .whereIn('size', listedOnce)
            .whereNull('md5_16k')
            .whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
              binary.posted_at,
              PAR2_MATCH_WINDOW_SECONDS,
            ])
            .select('id', 'release_id', 'size')
        : []
      const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
      const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
      const candidates = [...exact, ...fuzzy]

      // Prefer a release the exactly matched files are already in (e.g.
      // obfuscated volumes grouped by their stem); otherwise the set's own.
      const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
      let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
      const identity = par2ReleaseIdentity(
        set.recoverySetId,
        set.files.map((f) => f.name)
      )
      if (!releaseId) {
        const ids = await IngestService.upsertReleases(trx.knexClient, [
          {
            release: identity,
            poster: binary.poster,
            weakIdentity: true,
            groups: binary.groups,
            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
          },
        ])
        releaseId = ids.get(identity.key)!
      }

      await trx
        .knexClient('release_files')
        .insert(
          set.files.map((f) => ({
            release_id: releaseId,
            par2_file_id: f.fileId,
            name: f.name,
            size: f.length,
            md5_16k: f.md5_16k,
            md5: f.md5,
            set_id: set.recoverySetId,
          }))
        )
        .onConflict(['release_id', 'par2_file_id'])
        .ignore()

      const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
      await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
      // Exact matches sitting alone in a random-named release (one file of a
      // season, each posted under its own random name) join the set's release.
      await trx.rawQuery(
        `UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
        [releaseId, exact.map((c) => Number(c.id)), releaseId]
      )
      await trx.rawQuery(
        `UPDATE files f SET release_id = b.release_id FROM binaries b
         WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
        [releaseId]
      )

      // The set may have joined a release named after one random filename.
      let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
      const better = par2ReleaseName(set.files.map((f) => f.name))
      if (releaseName && better && isBetterName(releaseName, better)) {
        await trx.from('releases').where('id', releaseId).update({ name: better })
        releaseName = better
      }
      logger.info(
        `Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
          `${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
      )
      return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
    })

    // Real names for every known file of the set (exact fingerprint, or size within the release).
    await NameService.resolveForRelease(result.releaseId)
    return result
  }
}

export default new Par2Service()