The par2 merge: releases whose files one par2 set describes become one

The prober stores, per probed article that starts a file, the MD5 of its
first 16 KiB (file_hashes), and per par2 body the set's whole file list
(par2_files, par2_sets): FileDesc holds the same hash per file, whatever
the file was renamed to. releases:merge (the merger service, no NNTP)
merges every minute the releases of one group, posted within 48 hours of
a release holding the set's par2, whose files the set describes, into the
one with the most files: chunks, probe candidates, inspections, hashes and
counts move; the par2's shared name when none was readable; release_merges
keeps what went where. Obfuscated uploads whose files were each a release
of their own ("[9/20] - "14f5999a…"", hep: 13,232) become one.

Also: corpus_sweep.py counts the marked ranges without psql's command tag.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/functional/par2_merger.spec.ts

@@ -0,0 +1,65 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import { Par2Merger } from '#services/par2_merger'
+
+const hash = (c: string) => c.repeat(32)
+
+test.group('Par2Merger', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+
+  test('merges the one-file releases of an obfuscated upload by its par2 set, named by the par2', async ({ assert }) => {
+    const at = new Date('2022-07-08T03:46:00Z')
+    const release = async (key: string, posted: Date, groupId: number, files = 1) => {
+      const [row] = await db
+        .table('releases')
+        .insert({
+          release_key: key.padEnd(40, '0'), identity_key: key.padEnd(40, '0'), name: key, source: 'stem', groups: '[]', group_id: groupId,
+          files, segments: files * 10, bytes: files * 1000, first_article: 100, last_article: 200, posted_at: posted, readable: false, file_total: 20,
+          settled_at: posted,
+        })
+        .returning('id')
+      return Number(row.id)
+    }
+    // The par2 index, two data files of the upload, the same file reposted a week later, and one in another group.
+    const par2 = await release('14f5999a', at, 1)
+    const a = await release('2b7c00e1', new Date(at.getTime() + 60_000), 1)
+    const b = await release('9d01aa3f', new Date(at.getTime() + 120_000), 1, 2)
+    const later = await release('5e5e5e5e', new Date(at.getTime() + 7 * 86400_000), 1)
+    const elsewhere = await release('6f6f6f6f', at, 2)
+    let candidate = 1000
+    const hashed = async (releaseId: number, h: string, size: number) => {
+      candidate++
+      await db.table('probe_candidates').insert({ id: candidate, release_id: releaseId, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
+      await db.table('file_hashes').insert({ candidate_id: candidate, release_id: releaseId, hash16k: h, size })
+    }
+    await hashed(a, hash('a'), 104857600)
+    await hashed(b, hash('b'), 104857600)
+    await hashed(later, hash('a'), 104857600)
+    await hashed(elsewhere, hash('b'), 104857600)
+    // A different size: not the same file.
+    await hashed(await release('7a7a7a7a', at, 1), hash('a'), 5)
+    const setId = 'c'.repeat(32)
+    await db.table('par2_files').multiInsert([
+      { set_id: setId, file_id: 'd'.repeat(32), hash16k: hash('a'), length: 104857600, name: 'Some.Movie.2022.1080p.part01.rar' },
+      { set_id: setId, file_id: 'e'.repeat(32), hash16k: hash('b'), length: 104857600, name: 'Some.Movie.2022.1080p.part02.rar' },
+    ])
+    await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 })
+
+    const round = await new Par2Merger().round()
+    assert.deepEqual(round, { sets: 1, merges: 1, releases: 3 })
+    const target = await db.from('releases').where('id', b).firstOrFail()
+    assert.equal(target.files, 4)
+    assert.equal(target.segments, 40)
+    assert.equal(target.name, 'Some.Movie.2022.1080p')
+    assert.isTrue(target.complete === false)
+    assert.deepEqual((await db.from('releases').whereIn('id', [par2, a]).select('id')).length, 0)
+    assert.equal((await db.from('releases').whereIn('id', [later, elsewhere]).select('id')).length, 2)
+    assert.deepEqual((await db.from('probe_candidates').where('release_id', b).orderBy('id').select('id')).map((r) => Number(r.id)), [1001, 1002])
+    assert.equal((await db.from('par2_sets').where('set_id', setId).firstOrFail()).release_id, String(b))
+    assert.equal((await db.from('release_merges').where('into_id', b).count('* as n'))[0].n, '2')
+
+    // Again: nothing new.
+    assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0)
+  })
+})

tests/unit/probe/inspect.spec.ts

@@ -80,7 +80,28 @@ test.group('probe / inspect', () => {
     const setId = Buffer.alloc(16, 3)
     const par2 = Buffer.concat([fileDesc(setId, 'Movie.2020.part02.rar', 100), fileDesc(setId, 'Movie.2020.part01.rar', 200)])
     const result = inspect(article(par2, 'a8f3k2.par2'))
-    assert.deepEqual(result, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
+    const { par2: set, hash16k, ...rest } = result
+    assert.deepEqual(rest, { yencName: 'a8f3k2.par2', size: par2.length, type: 'par2', entries: 2, names: ['Movie.2020.part01.rar', 'Movie.2020.part02.rar'] })
+    // The whole set for the par2 merge: per file the hash of its first 16 KiB, its length and name.
+    assert.equal(set!.setId, setId.toString('hex'))
+    assert.deepEqual(
+      set!.files.map((f) => [f.name, f.length, f.hash16k]),
+      [
+        ['Movie.2020.part02.rar', 100, md5(Buffer.from('Movie.2020.part02.rar-16k')).toString('hex')],
+        ['Movie.2020.part01.rar', 200, md5(Buffer.from('Movie.2020.part01.rar-16k')).toString('hex')],
+      ]
+    )
+    assert.equal(hash16k, md5(par2).toString('hex'))
+  })
+
+  test("the hash of a file's first 16 KiB, when the article starts the file", ({ assert }) => {
+    const data = Buffer.alloc(40_000, 7)
+    assert.equal(inspect(article(data, 'x')).hash16k, md5(data.subarray(0, 16384)).toString('hex'))
+    const part = Buffer.from(
+      [`=ybegin part=2 total=3 line=128 size=120000 name=x`, `=ypart begin=40001 end=80000`, yencode.encode(data).toString('latin1'), `=yend size=40000 part=2 pcrc32=${crc32Hex(data)}`].join('\r\n'),
+      'latin1'
+    )
+    assert.isUndefined(inspect(part).hash16k)
   })
 
   test('a RAR volume: its entries, and whether they are encrypted', ({ assert }) => {
@@ -96,7 +117,7 @@ test.group('probe / inspect', () => {
 
   test('other data: the yEnc name and what the bytes are', ({ assert }) => {
     const mkv = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3]), Buffer.alloc(100)])
-    assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska' })
+    assert.deepEqual(inspect(article(mkv, 'Real.Name.mkv')), { yencName: 'Real.Name.mkv', size: 104, type: 'matroska', hash16k: md5(mkv).toString('hex') })
   })
 
   test('not yEnc: throws', ({ assert }) => {