implemented a lot of features

tests/functional/split.spec.ts

@@ -0,0 +1,84 @@
+import { test } from '@japa/runner'
+import db from '@adonisjs/lucid/services/db'
+import testUtils from '@adonisjs/core/services/test_utils'
+import IngestService, { type IngestSegment } from '#services/IngestService'
+import SplitService from '#services/SplitService'
+import { weakFileKey } from '#lib/subject/index'
+import { crc32Hex } from '#lib/yenc/index'
+
+// Three 4-byte parts per volume, with real CRCs so the split volumes verify.
+const DATA: Record<string, Buffer> = { A: Buffer.from('AAAAaaaa1111'), B: Buffer.from('BBBBbbbb2222') }
+
+const KEY = weakFileKey('a8f3Kq', 10_000_952, 3)
+
+function article(volume: string, number: number, articleNumber: number): IngestSegment {
+  return {
+    fileKey: KEY,
+    poster: `random-${articleNumber}@example.invalid`,
+    subject: '"a8f3Kq" yEnc',
+    name: `random${articleNumber}`,
+    totalParts: 3,
+    number,
+    messageId: `${volume}-${number}@test`,
+    bytes: 700_000,
+    articleNumber,
+    groups: ['alt.binaries.test'],
+    weakIdentity: true,
+    size: 10_000_952,
+    release: null,
+    pcrc32: crc32Hex(DATA[volume].subarray((number - 1) * 4, number * 4)),
+    partSize: 4,
+    fileCrc32: number === 3 ? crc32Hex(DATA[volume]) : undefined,
+  }
+}
+
+test.group('SplitService', (group) => {
+  group.each.setup(() => testUtils.db().withGlobalTransaction())
+  group.each.setup(() => {
+    // Keep completed volumes out of the real file queue.
+    const enqueue = IngestService.enqueue
+    IngestService.enqueue = async () => {}
+    return () => {
+      IngestService.enqueue = enqueue
+    }
+  })
+
+  test('splits two equal-sized volumes into verified, complete files', async ({ assert }) => {
+    // Volume A then volume B, arriving interleaved and out of order.
+    await IngestService.ingest([article('A', 1, 101), article('A', 2, 102), article('B', 1, 104)])
+    await IngestService.ingest([article('B', 2, 105), article('A', 3, 103), article('B', 3, 106)])
+
+    const parent = await db.from('binaries').where('file_key', KEY).first()
+    assert.isNotNull(parent.conflicted_at)
+
+    const result = await SplitService.splitPending()
+    assert.equal(result.volumes, 2)
+    assert.equal(result.completed, 2)
+
+    const volumes = await db.from('binaries').where('parent_binary_id', parent.id).orderBy('id')
+    assert.lengthOf(volumes, 2)
+    for (const [i, volume] of volumes.entries()) {
+      assert.isNotNull(volume.completed_at)
+      const parts = await db.from('parts').where('binary_id', volume.id).orderBy('number').select('message_id')
+      const letter = i === 0 ? 'A' : 'B'
+      assert.deepEqual(parts.map((p) => p.message_id), [1, 2, 3].map((n) => `${letter}-${n}@test`))
+    }
+
+    // Nothing new since the split: the next run leaves it alone.
+    assert.equal((await SplitService.splitPending()).parents, 0)
+  })
+
+  test('a mix that fails CRC verification is not turned into a file', async ({ assert }) => {
+    // B's last part is missing: the only complete candidate would mix volumes.
+    await IngestService.ingest([article('A', 1, 101), article('B', 1, 102), article('A', 2, 103)])
+    await IngestService.ingest([article('B', 2, 104), article('A', 3, 105)])
+    const parent = await db.from('binaries').where('file_key', KEY).first()
+    const result = await SplitService.splitPending()
+    const volumes = await db.from('binaries').where('parent_binary_id', parent.id)
+    for (const volume of volumes) {
+      const parts = await db.from('parts').where('binary_id', volume.id).select('message_id')
+      assert.lengthOf(new Set(parts.map((p) => p.message_id[0])), 1)
+    }
+    assert.isAtMost(result.completed, 1)
+  })
+})