The par2 merge: releases whose files one par2 set describes become one

The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.

Also: corpus_sweep.py counts the marked ranges without psql's command tag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/probe/inspect.spec.ts

@@ -80,7 +80,28 @@ test.group('probe / inspect', () => {
     const setId = Buffer.alloc(16, 3)
     const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)])
     const result = inspect(article(par2, 'a8f3k2.par2'))
-    assert.deepEqual(result, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
+    const { par2: set, hash16k, ...rest } = result
+    assert.deepEqual(rest, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
+    // The whole set for the par2 merge: per file the hash of its first 16 KiB, its length and name.
+    assert.equal(set!.setId, setId.toString('hex'))
+    assert.deepEqual(
+      set!.files.map((f) => [f.name, f.length, f.hash16k]),
+      [
+        ['Movie.2020.part02.rar', 100, md5(Buffer.from('Movie.2020.part02.rar-16k')).toString('hex')],
+        ['Movie.2020.part01.rar', 200, md5(Buffer.from('Movie.2020.part01.rar-16k')).toString('hex')],
+      ]
+    )
+    assert.equal(hash16k, md5(par2).toString('hex'))
+  })
+
+  test("the hash of a file's first 16 KiB, when the article starts the file", ({ assert }) => {
+    const data = Buffer.alloc(40_000, 7)
+    assert.equal(inspect(article(data, 'x')).hash16k, md5(data.subarray(0, 16384)).toString('hex'))
+    const part = Buffer.from(
+      [`=ybegin part=2 total=3 line=128 size=120000 name=x`, `=ypart begin=40001 end=80000`, yencode.encode(data).toString('latin1'), `=yend size=40000 part=2 pcrc32=${crc32Hex(data)}`].join('\r\n'),
+      'latin1'
+    )
+    assert.isUndefined(inspect(part).hash16k)
   })
 
   test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => {
@@ -96,7 +117,7 @@ test.group('probe / inspect', () => {
 
   test('other data: the yEnc name and what the bytes are', ({ assert }) => {
     const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)])
-    assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska' })
+    assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska', hash16k: md5(mkv).toString('hex') })
   })
 
   test('not yEnc: throws', ({ assert }) => {