implemented a lot of features

app/services/Par2Service.ts

@@ -0,0 +1,165 @@
+import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
+import NntpService from '#services/NntpService'
+import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
+import { decodeSegment, YencFile } from '#lib/yenc/index'
+import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
+import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
+import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
+
+/** Par2 index files are small; recovery volumes can be large and are not needed. */
+const MAX_PARTS = 20
+/** Size-only matching below this is too ambiguous. */
+const SIZE_ONLY_MIN = 1024 * 1024
+
+function countBy<T>(values: T[]): Map<T, number> {
+  const counts = new Map<T, number>()
+  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
+  return counts
+}
+
+export interface Par2Result {
+  releaseId: number
+  releaseName: string
+  listed: number
+  matched: number
+}
+
+/**
+ * Recognises releases of obfuscated posts from their par2 files: the par2
+ * lists the real name and exact size of every file in the set, and
+ * body-derived binaries carry their exact size. See CONCEPT.md (D16).
+ */
+class Par2Service {
+  /**
+   * Reads a par2 file whose first segment is already decoded, downloading the
+   * remaining parts if there are any. Returns null when it is not par2.
+   */
+  async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
+    if (!isPar2(first)) return null
+    const ids = Object.entries(messageIds)
+      .sort(([a], [b]) => Number(a) - Number(b))
+      .map(([, part]) => part.id)
+    if (ids.length > MAX_PARTS) return null
+    if (ids.length === 1) return parsePar2(first)
+
+    let assembled: YencFile | undefined
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      for (const id of ids) {
+        const segment = decodeSegment(await NntpService.fetchBody(conn, id))
+        assembled ??= YencFile.fromSegment(segment)
+        assembled.add(segment)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    return parsePar2(assembled!.getBuffer())
+  }
+
+  async attach(binary: any, set: Par2Set): Promise<Par2Result> {
+    const result = await db.transaction(async (trx) => {
+      // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
+      // the set into a release the files already belong to; size alone is
+      // ambiguous across posts (standard volume sizes recur).
+      const exact = await trx
+        .from('binaries')
+        .whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
+        .select('id', 'release_id')
+
+      // Size only, for files without a fingerprint: the size must be listed
+      // once in this set, be large, and belong to exactly one binary nearby.
+      const listedOnce = [...countBy(set.files.map((f) => f.length))]
+        .filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
+        .map(([size]) => size)
+      const bySize = listedOnce.length
+        ? await trx
+            .from('binaries')
+            .whereIn('size', listedOnce)
+            .whereNull('md5_16k')
+            .whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
+              binary.posted_at,
+              PAR2_MATCH_WINDOW_SECONDS,
+            ])
+            .select('id', 'release_id', 'size')
+        : []
+      const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
+      const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
+      const candidates = [...exact, ...fuzzy]
+
+      // Prefer a release the exactly matched files are already in (e.g.
+      // obfuscated volumes grouped by their stem); otherwise the set's own.
+      const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
+      let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
+      const identity = par2ReleaseIdentity(
+        set.recoverySetId,
+        set.files.map((f) => f.name)
+      )
+      if (!releaseId) {
+        const ids = await IngestService.upsertReleases(trx.knexClient, [
+          {
+            release: identity,
+            poster: binary.poster,
+            weakIdentity: true,
+            groups: binary.groups,
+            postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
+          },
+        ])
+        releaseId = ids.get(identity.key)!
+      }
+
+      await trx
+        .knexClient('release_files')
+        .insert(
+          set.files.map((f) => ({
+            release_id: releaseId,
+            par2_file_id: f.fileId,
+            name: f.name,
+            size: f.length,
+            md5_16k: f.md5_16k,
+            md5: f.md5,
+            set_id: set.recoverySetId,
+          }))
+        )
+        .onConflict(['release_id', 'par2_file_id'])
+        .ignore()
+
+      const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
+      await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
+      // Exact matches sitting alone in a random-named release (one file of a
+      // season, each posted under its own random name) join the set's release.
+      await trx.rawQuery(
+        `UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
+        [releaseId, exact.map((c) => Number(c.id)), releaseId]
+      )
+      await trx.rawQuery(
+        `UPDATE files f SET release_id = b.release_id FROM binaries b
+         WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
+        [releaseId]
+      )
+
+      // The set may have joined a release named after one random filename.
+      let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
+      const better = par2ReleaseName(set.files.map((f) => f.name))
+      if (releaseName && better && isBetterName(releaseName, better)) {
+        await trx.from('releases').where('id', releaseId).update({ name: better })
+        releaseName = better
+      }
+      logger.info(
+        `Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
+          `${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
+      )
+      return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
+    })
+
+    // Real names for every known file of the set (exact fingerprint, or size within the release).
+    await NameService.resolveForRelease(result.releaseId)
+    return result
+  }
+}
+
+export default new Par2Service()