import { createHash } from 'node:crypto' import db from '@adonisjs/lucid/services/db' import IngestService from '#services/IngestService' import NameService from '#services/NameService' import { splitVerified } from '#lib/split/index' /** Conflicted binaries handled per run; the rest follow on the next tick. */ const BATCH = 200 export interface SplitResult { parents: number volumes: number completed: number } /** * Splits conflicted obfuscated binaries (several equal-sized files under one * size+parts identity) into one binary per volume. Only volumes whose * combined part CRCs match the file CRC of their last part are created, so a * split can never produce a corrupt file; the rest waits for more articles. * See CONCEPT.md (D20). */ class SplitService { async splitPending(): Promise { const parents = await db .from('binaries') .whereNotNull('conflicted_at') .whereNull('parent_binary_id') // split_at = 'infinity' marks binaries that cannot be split (see migration …810). .where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at')) // Most recently active first: those are the posts still coming in. .orderBy('last_seen_at', 'desc') .limit(BATCH) const volumeIds: number[] = [] for (const parent of parents) { volumeIds.push(...(await this.split(parent))) } // Former conflicts that turned out to be reposts (unflagged, keep their // split_at): no new article may arrive to trigger their completion. const unflagged = await db .from('binaries') .whereNull('conflicted_at') .whereNull('completed_at') .whereNull('parent_binary_id') .whereNotNull('split_at') .whereRaw("split_at <> 'infinity'") .limit(BATCH) .select('id') volumeIds.push(...unflagged.map((b) => Number(b.id))) await IngestService.matchPar2Releases(volumeIds) await NameService.resolve(volumeIds) const completed = await IngestService.claimCompleted(volumeIds) await IngestService.enqueue(completed) return { parents: parents.length, volumes: volumeIds.length, completed: completed.length } } /** Creates the newly verified volumes of one conflicted binary; returns their ids. */ private async split(parent: any): Promise { return db.transaction(async (trx) => { // Unverified volumes from before verification existed can never complete. await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete() const { rows } = await trx.rawQuery( `WITH taken AS ( SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id WHERE v.parent_binary_id = ? ), candidates AS ( SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ? UNION ALL SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ? ) SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`, [parent.id, parent.id, parent.id] ) const volumes = splitVerified( rows.map((r: any) => ({ number: r.number, messageId: r.message_id, bytes: r.bytes, articleNumber: r.article_number === null ? null : Number(r.article_number), pcrc32: r.pcrc32, partSize: r.part_size, fileCrc32: r.file_crc32, })), { totalParts: parent.total_parts } ) const now = new Date() const ids: number[] = [] for (const articles of volumes) { // A verified volume is fixed, so its first article identifies it. const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex') const inserted = await trx .knexClient('binaries') .insert({ file_key: fileKey, poster: parent.poster, subject: parent.subject, name: parent.name, total_parts: parent.total_parts, groups: JSON.stringify(parent.groups), posted_at: parent.posted_at, size: parent.size, release_id: parent.release_id, parent_binary_id: parent.id, first_seen_at: now, last_seen_at: now, }) .onConflict('file_key') .ignore() .returning('id') if (inserted.length === 0) continue const id = Number(inserted[0].id) await trx.knexClient('parts').insert( articles.map((a) => ({ binary_id: id, number: a.number, message_id: a.messageId, bytes: a.bytes, article_number: a.articleNumber, pcrc32: a.pcrc32, part_size: a.partSize, file_crc32: a.fileCrc32, })) ) // The fingerprint of a volume is that of its part 1 article. await trx.rawQuery( `UPDATE binaries SET md5_16k = ( SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ? UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1) WHERE id = ?`, [parent.id, articles[0].messageId, parent.id, articles[0].messageId, id] ) ids.push(id) } await trx.from('binaries').where('id', parent.id).update({ split_at: now }) return ids }) } } export default new SplitService()