The par2 merge: releases whose files one par2 set describes become one

The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.

Also: corpus_sweep.py counts the marked ranges without psql's command tag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/probe/inspect.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto'
 import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
 import { isPar2, parsePar2 } from '#lib/par2/index'
 import { parseRarHeaders } from '#lib/rar/index'
@@ -16,8 +17,17 @@ export interface Inspection {
   entries?: number
   /** An archive with encrypted entries or headers. */
   passworded?: boolean
+  /**
+   * MD5 of the file's first 16 KiB (hex), when this article starts the file: what a par2 set's FileDesc
+   * holds per file, so a renamed file is recognised by it (the par2 merge; app/services/par2_merger.ts).
+   */
+  hash16k?: string
+  /** A par2 body's recovery set: every file it describes (not kept in release_inspections; par2_files). */
+  par2?: { setId: string; files: { fileId: string; hash16k: string; length: number; name: string }[] }
 }
 
+const HASH_BYTES = 16 * 1024
+
 /** Names kept per inspection: the catalog wants the content's name, not every volume's. */
 export const MAX_NAMES = 8
 const MAX_NAME_LENGTH = 200
@@ -46,12 +56,20 @@ function largestFirst(items: { name: string; size: number }[]): string[] {
 export function inspect(body: Buffer): Inspection {
   const segment = decodeBody(body)
   const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
+  // The file's first 16 KiB, or all of a smaller file.
+  if (segment.offset === 0 && (segment.data.length >= HASH_BYTES || segment.data.length === result.size)) {
+    result.hash16k = createHash('md5').update(segment.data.subarray(0, HASH_BYTES)).digest('hex')
+  }
   if (isPar2(segment.data)) {
     // An index file fits one article; what a larger one cut off is skipped by the parser.
     const set = parsePar2(segment.data)
     if (set) {
       result.entries = set.files.length
       result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
+      result.par2 = {
+        setId: set.recoverySetId,
+        files: set.files.map((f) => ({ fileId: f.fileId, hash16k: f.md5_16k, length: f.length, name: f.name.slice(0, MAX_NAME_LENGTH) })),
+      }
     }
     return result
   }