implemented a lot of features

app/services/SplitService.ts

@@ -0,0 +1,146 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import IngestService from '#services/IngestService'
+import NameService from '#services/NameService'
+import { splitVerified } from '#lib/split/index'
+
+/** Conflicted binaries handled per run; the rest follow on the next tick. */
+const BATCH = 200
+
+export interface SplitResult {
+  parents: number
+  volumes: number
+  completed: number
+}
+
+/**
+ * Splits conflicted obfuscated binaries (several equal-sized files under one
+ * size+parts identity) into one binary per volume. Only volumes whose
+ * combined part CRCs match the file CRC of their last part are created, so a
+ * split can never produce a corrupt file; the rest waits for more articles.
+ * See CONCEPT.md (D20).
+ */
+class SplitService {
+  async splitPending(): Promise<SplitResult> {
+    const parents = await db
+      .from('binaries')
+      .whereNotNull('conflicted_at')
+      .whereNull('parent_binary_id')
+      // split_at = 'infinity' marks binaries that cannot be split (see migration …810).
+      .where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
+      // Most recently active first: those are the posts still coming in.
+      .orderBy('last_seen_at', 'desc')
+      .limit(BATCH)
+
+    const volumeIds: number[] = []
+    for (const parent of parents) {
+      volumeIds.push(...(await this.split(parent)))
+    }
+
+    // Former conflicts that turned out to be reposts (unflagged, keep their
+    // split_at): no new article may arrive to trigger their completion.
+    const unflagged = await db
+      .from('binaries')
+      .whereNull('conflicted_at')
+      .whereNull('completed_at')
+      .whereNull('parent_binary_id')
+      .whereNotNull('split_at')
+      .whereRaw("split_at <> 'infinity'")
+      .limit(BATCH)
+      .select('id')
+    volumeIds.push(...unflagged.map((b) => Number(b.id)))
+
+    await IngestService.matchPar2Releases(volumeIds)
+    await NameService.resolve(volumeIds)
+    const completed = await IngestService.claimCompleted(volumeIds)
+    await IngestService.enqueue(completed)
+    return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
+  }
+
+  /** Creates the newly verified volumes of one conflicted binary; returns their ids. */
+  private async split(parent: any): Promise<number[]> {
+    return db.transaction(async (trx) => {
+      // Unverified volumes from before verification existed can never complete.
+      await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()
+
+      const { rows } = await trx.rawQuery(
+        `WITH taken AS (
+           SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
+           WHERE v.parent_binary_id = ?
+         ), candidates AS (
+           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
+           UNION ALL
+           SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
+         )
+         SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
+        [parent.id, parent.id, parent.id]
+      )
+      const volumes = splitVerified(
+        rows.map((r: any) => ({
+          number: r.number,
+          messageId: r.message_id,
+          bytes: r.bytes,
+          articleNumber: r.article_number === null ? null : Number(r.article_number),
+          pcrc32: r.pcrc32,
+          partSize: r.part_size,
+          fileCrc32: r.file_crc32,
+        })),
+        { totalParts: parent.total_parts }
+      )
+
+      const now = new Date()
+      const ids: number[] = []
+      for (const articles of volumes) {
+        // A verified volume is fixed, so its first article identifies it.
+        const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
+        const inserted = await trx
+          .knexClient('binaries')
+          .insert({
+            file_key: fileKey,
+            poster: parent.poster,
+            subject: parent.subject,
+            name: parent.name,
+            total_parts: parent.total_parts,
+            groups: JSON.stringify(parent.groups),
+            posted_at: parent.posted_at,
+            size: parent.size,
+            release_id: parent.release_id,
+            parent_binary_id: parent.id,
+            first_seen_at: now,
+            last_seen_at: now,
+          })
+          .onConflict('file_key')
+          .ignore()
+          .returning('id')
+        if (inserted.length === 0) continue
+        const id = Number(inserted[0].id)
+        await trx.knexClient('parts').insert(
+          articles.map((a) => ({
+            binary_id: id,
+            number: a.number,
+            message_id: a.messageId,
+            bytes: a.bytes,
+            article_number: a.articleNumber,
+            pcrc32: a.pcrc32,
+            part_size: a.partSize,
+            file_crc32: a.fileCrc32,
+          }))
+        )
+        // The fingerprint of a volume is that of its part 1 article.
+        await trx.rawQuery(
+          `UPDATE binaries SET md5_16k = (
+             SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
+             UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
+           WHERE id = ?`,
+          [parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
+        )
+        ids.push(id)
+      }
+
+      await trx.from('binaries').where('id', parent.id).update({ split_at: now })
+      return ids
+    })
+  }
+}
+
+export default new SplitService()