import db from '@adonisjs/lucid/services/db' import FileStore from '#services/FileStore' import SfvService from '#services/SfvService' import ContentService from '#services/ContentService' import InspectionQueue from '#services/InspectionQueue' export interface BinaryFilesResult { /** New files. */ saved: number /** Binaries whose file already existed (same first segment or file_key). */ existing: number /** Conflicted, gone or already linked: nothing to do. */ skipped: number /** Not all parts stored yet (the job is retried for these). */ incomplete: number[] inspections: number } /** * Turns completed binaries into files (worker:file), a batch at a time * (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and * files that exist already, one INSERT for the new files (so the files * statement trigger updates each release once), and one UPDATE and DELETE * for the binaries' links and copied parts. It was one job of ~8 queries per * binary, ~700 a minute. */ class BinaryFileService { async store(binaryIds: number[]): Promise { const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 } const ids = [...new Set(binaryIds)] const binaries = await db .from('binaries') .whereIn('id', ids) .whereNull('conflicted_at') .whereNull('file_id') .select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts') result.skipped = ids.length - binaries.length if (binaries.length === 0) return result const partsOf = new Map() const parts = await db .from('parts') .whereIn('binary_id', binaries.map((b) => b.id)) .orderBy([{ column: 'binary_id' }, { column: 'number' }]) .select('binary_id', 'number', 'message_id', 'bytes') for (const part of parts) { const list = partsOf.get(Number(part.binary_id)) ?? [] list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) }) partsOf.set(Number(part.binary_id), list) } const complete = binaries.filter((b) => { const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts if (!ok) result.incomplete.push(Number(b.id)) return ok }) if (complete.length === 0) return result // Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list). const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id) const byFirst = new Map() for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) { byFirst.set(row.first_message_id, Number(row.id)) } const fileIdOf = new Map() const fresh = complete.filter((b) => { const existing = byFirst.get(partsOf.get(Number(b.id))![0].id) if (existing) fileIdOf.set(Number(b.id), existing) return !existing }) const now = new Date() // file_key is unique: a second completion of the same upload inserts nothing. const inserted = await FileStore.insert( fresh.map((b) => { const segments = partsOf.get(Number(b.id))! return { file_key: b.file_key, filename: b.name, poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(), parts: segments.length, segments, groups: JSON.stringify(b.groups), release_id: b.release_id, real_name: b.real_name, size: segments.reduce((sum, s) => sum + s.size, 0), created_at: now, updated_at: now, } }) ) const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id])) const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key) const byKey = new Map() if (missing.length > 0) { for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id)) } for (const b of fresh) { const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key) if (id) fileIdOf.set(Number(b.id), id) } const fresher = new Map(fresh.map((b) => [b.file_key, b])) result.saved = inserted.length result.existing = fileIdOf.size - inserted.length // Links, locked in file_key order like ingest's upsert, so the two can't deadlock. const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key)) if (linked.length > 0) { await db.transaction(async (trx) => { await trx.rawQuery( `SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`, [linked.map((b) => b.id)] ) // Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16). const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id)) await trx.rawQuery( `UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`, [linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))] ) }) // Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20). await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)]) } // As for one file before: an SFV posted earlier may list them (D34), then inspection. const added = inserted.flatMap((row) => { const b = row.file_key ? fresher.get(row.file_key) : undefined return b ? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }] : [] }) if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file)) // By the posted name, as before: inspection skips later volumes and readable media names anyway. const jobs = added .filter((file) => ContentService.isCandidate(file.posted)) .map((file) => file.id) if (jobs.length > 0) await InspectionQueue.add(jobs) result.inspections = jobs.length return result } } export default new BinaryFileService()