import db from '@adonisjs/lucid/services/db' import type { AggregatedFile } from '#lib/aggregate/index' import IngestService, { type IngestSegment } from '#services/IngestService' import FileStore from '#services/FileStore' import SfvService from '#services/SfvService' import ContentService from '#services/ContentService' import QueueService from '#services/QueueService' export interface SeedStoreResult { files: number /** Files that existed already (same file_key, e.g. from live indexing or a crosspost). */ existing: number releases: number inspections: number } /** * Stores files an edge node aggregated from headers (CONCEPT.md D37): their * releases, the files with their segment lists, and one completed binary per * file, as worker:file would have left it, so par2 naming, inspection and GC * treat them like any other file. A batch is a few statements, not one per * article. */ class SeedService { async store(files: AggregatedFile[]): Promise { const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0 } if (files.length === 0) return result const knex = db.connection().getWriteClient() const now = new Date() // A file's identity and release come from its first segment, its date from the earliest. const firstOf = (file: AggregatedFile) => file.segments[0] const postedOf = (file: AggregatedFile) => { const dates = file.segments.map((s) => s.postedAt?.getTime()).filter((t): t is number => Number.isFinite(t)) return dates.length > 0 ? new Date(Math.min(...dates)) : undefined } const releaseIds = await IngestService.upsertReleases( knex, files.map((file) => ({ ...firstOf(file), postedAt: postedOf(file) })), now ) result.releases = releaseIds.size const rows = files.map((file) => { const first = firstOf(file) const postedAt = postedOf(file) return { file_key: file.fileKey, filename: first.name, poster: first.poster, date: (postedAt ?? now).getTime(), parts: file.segments.length, segments: file.segments.map((s) => ({ number: s.number, id: s.messageId, size: s.bytes })), groups: JSON.stringify(first.groups), release_id: first.release ? (releaseIds.get(first.release.key) ?? null) : null, size: file.segments.reduce((sum, s) => sum + s.bytes, 0), created_at: now, updated_at: now, } }) const inserted = await FileStore.insert(rows) result.files = inserted.length result.existing = rows.length - inserted.length const fileIds = new Map() for (const row of await db.from('files').whereIn('file_key', rows.map((r) => r.file_key)).select('id', 'file_key')) { fileIds.set(row.file_key, Number(row.id)) } // The completed binary worker:file would have left behind; an existing one just learns its file. const binaries = rows.map((row, i) => ({ file_key: row.file_key, poster: row.poster, subject: firstOf(files[i]).subject, name: row.filename, total_parts: firstOf(files[i]).totalParts, groups: row.groups, posted_at: postedOf(files[i]) ?? null, first_seen_at: now, last_seen_at: now, completed_at: now, release_id: row.release_id, file_id: fileIds.get(row.file_key) ?? null, })) binaries.sort((a, b) => a.file_key.localeCompare(b.file_key)) await knex('binaries') .insert(binaries) .onConflict('file_key') .merge({ file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any }) // As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection. const rowById = new Map(rows.map((row) => [fileIds.get(row.file_key), row])) const candidates: { name: string; data: { fileId: number }; opts: { jobId: string } }[] = [] for (const file of inserted) { const row = rowById.get(file.id) if (!row) continue await SfvService.claim({ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }) if (ContentService.isCandidate(row.filename)) { candidates.push({ name: 'process-collection', data: { fileId: file.id }, opts: { jobId: `file-${file.id}` } }) } } if (candidates.length > 0) await QueueService.collectionQueue.addBulk(candidates) result.inspections = candidates.length return result } } export default new SeedService()