import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService' import type { Par2Set } from '#lib/par2/index' import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index' import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService' /** Size-only matching below this is too ambiguous. */ const SIZE_ONLY_MIN = 1024 * 1024 function countBy(values: T[]): Map { const counts = new Map() for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1) return counts } export interface Par2Result { releaseId: number releaseName: string listed: number matched: number } /** * Recognises releases of obfuscated posts from their par2 files: the par2 * lists the real name and exact size of every file in the set, and * body-derived binaries carry their exact size. See CONCEPT.md (D16). */ class Par2Service { async attach(binary: any, set: Par2Set): Promise { const result = await db.transaction(async (trx) => { // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull // the set into a release the files already belong to; size alone is // ambiguous across posts (standard volume sizes recur). const exact = await trx .from('binaries') .whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any) .select('id', 'release_id') // Size only, for files without a fingerprint: the size must be listed // once in this set, be large, and belong to exactly one binary nearby. const listedOnce = [...countBy(set.files.map((f) => f.length))] .filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN) .map(([size]) => size) const bySize = listedOnce.length ? await trx .from('binaries') .whereIn('size', listedOnce) .whereNull('md5_16k') .whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [ binary.posted_at, PAR2_MATCH_WINDOW_SECONDS, ]) .select('id', 'release_id', 'size') : [] const sizeCounts = countBy(bySize.map((b) => Number(b.size))) const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1) const candidates = [...exact, ...fuzzy] // Prefer a release the exactly matched files are already in (e.g. // obfuscated volumes grouped by their stem); otherwise the set's own. const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id))) let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] const identity = par2ReleaseIdentity( set.recoverySetId, set.files.map((f) => f.name) ) if (!releaseId) { const ids = await IngestService.upsertReleases(trx.knexClient, [ { release: identity, poster: binary.poster, weakIdentity: true, groups: binary.groups, postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined, }, ]) releaseId = ids.get(identity.key)! } await trx .knexClient('release_files') .insert( set.files.map((f) => ({ release_id: releaseId, par2_file_id: f.fileId, name: f.name, size: f.length, md5_16k: f.md5_16k, md5: f.md5, set_id: set.recoverySetId, })) ) .onConflict(['release_id', 'par2_file_id']) .ignore() const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id)) await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId }) // Exact matches sitting alone in a random-named release (one file of a // season, each posted under its own random name) join the set's release. await trx.rawQuery( `UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`, [releaseId, exact.map((c) => Number(c.id)), releaseId] ) await trx.rawQuery( `UPDATE files f SET release_id = b.release_id FROM binaries b WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`, [releaseId] ) // The set may have joined a release named after one random filename. const release = await trx.from('releases').where('id', releaseId).first() let releaseName = release?.name const better = par2ReleaseName(set.files.map((f) => f.name)) // A spot's title is kept (CONCEPT.md D29). if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) { await trx.from('releases').where('id', releaseId).update({ name: better }) releaseName = better } logger.info( `Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` + `${set.files.length} files listed, ${matchIds.length} binaries matched by size.` ) return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length } }) // Real names for every known file of the set (exact fingerprint, or size within the release). await NameService.resolveForRelease(result.releaseId) return result } } export default new Par2Service()