implemented a lot of features

app/services/ContentService.ts

@@ -0,0 +1,187 @@
+import { createHash } from 'node:crypto'
+import db from '@adonisjs/lucid/services/db'
+import NntpService from '#services/NntpService'
+import Par2Service, { type Par2Result } from '#services/Par2Service'
+import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
+import NameService from '#services/NameService'
+import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
+import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
+import { sniffContentType, type ContentType } from '#lib/content/index'
+import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
+import { fileStem } from '#lib/release/index'
+import { looksLikeFilename } from '#lib/subject/index'
+
+const FIRST_CHUNK = 16 * 1024
+
+/**
+ * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
+ * the file's first part. Undefined when that part is shorter than 16 KiB but
+ * the file is not (the chunk would span two parts).
+ */
+export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
+  return chunkMd5(decodeSegment(body).data, fileSize)
+}
+
+function chunkMd5(data: Buffer, fileSize: number): string | undefined {
+  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
+  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
+}
+
+export interface ContentResult {
+  par2?: Par2Result
+  archive?: ArchiveResult
+  named: number
+  contentType: ContentType | null
+  nzb?: NzbImportResult
+  /** NZBs read from inside an archive. */
+  archiveNzbs?: NzbImportResult[]
+}
+
+/**
+ * Takes names and groupings from file contents (CONCEPT.md D21). For a
+ * completed file with a random name, one article (the first) is enough:
+ * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
+ * - par2 files are recognised by content and name every file of their set,
+ * - RAR headers give the release grouping, name and contents.
+ */
+class ContentService {
+  /** Random or missing names, and named first RAR volumes (for contents and password flag). */
+  isCandidate(name: string) {
+    if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
+    if (/\.nzb$/i.test(name)) return true
+    return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
+  }
+
+  /** Downloads a whole (small) file whose first segment is already decoded. */
+  private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
+    const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
+    const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
+    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
+    if (parts.length === 1) return first
+
+    let assembled: YencFile | undefined
+    const conn = await NntpService.acquire()
+    let failed = false
+    try {
+      for (const [, part] of parts) {
+        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
+        assembled ??= YencFile.fromSegment(segment)
+        assembled.add(segment)
+      }
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    return assembled!.getBuffer()
+  }
+
+  /**
+   * Downloads a small archive that lists NZBs and imports each of them
+   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
+   * uploads by other posters.
+   */
+  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
+    const archive = await this.readWhole(first, file.message_ids)
+    const info = archive && parseRarHeaders(archive)
+    if (!archive || !info) return []
+    const entries = info.entries.filter(isNzbEntry)
+    const results: NzbImportResult[] = []
+    for (const entry of entries) {
+      const xml = await extractEntry(archive, entry)
+      if (!xml) continue
+      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
+        entryName: entry.name,
+        moveContainer: entries.length === 1,
+      })
+      if (imported) results.push(imported)
+    }
+    return results
+  }
+
+  async inspect(fileId: number): Promise<ContentResult | null> {
+    const file = await db.from('files').where('id', fileId).first()
+    const binary = await db.from('binaries').where('file_id', fileId).first()
+    if (!file || !this.isCandidate(file.filename)) return null
+    const first = file.message_ids?.['1']
+    if (!first?.id) return null
+
+    const conn = await NntpService.acquire()
+    let failed = false
+    let body: Buffer
+    try {
+      body = await NntpService.fetchBody(conn, first.id)
+    } catch (error) {
+      failed = true
+      throw error
+    } finally {
+      NntpService.release(conn, failed)
+    }
+    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
+    let segment
+    try {
+      segment = decodeSegment(body)
+    } catch (error) {
+      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
+      segment = decodeSegment(body, { lenientSize: true })
+    }
+    const size = segment.meta.begin.size
+    const contentType = sniffContentType(segment.data)
+    await db.from('files').where('id', fileId).update({ content_type: contentType })
+
+    // Fingerprint, so par2 sets (now or later) can name this file exactly.
+    if (binary) {
+      await db
+        .from('binaries')
+        .where('id', binary.id)
+        .update({
+          size: db.raw('coalesce(size, ?)', [size]),
+          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
+        })
+    }
+
+    const result: ContentResult = { named: 0, contentType }
+    if (contentType === 'nzbe' || contentType === 'rclone') {
+      // An encrypted NZB, or rclone crypt (names and data encrypted): nothing
+      // to read, and useless without the key.
+      const releaseId = file.release_id ?? binary?.release_id
+      if (releaseId) {
+        await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
+      }
+      return result
+    }
+    if (contentType === 'nzb') {
+      const xml = await this.readWhole(segment.data, file.message_ids)
+      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
+      return result
+    }
+    const set = await Par2Service.read(segment.data, file.message_ids)
+    if (set && binary) {
+      result.par2 = await Par2Service.attach(binary, set)
+    } else {
+      const info = parseRarHeaders(segment.data)
+      if (info) {
+        result.archive = await ArchiveService.record(file, binary, info)
+        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
+      }
+    }
+    if (binary) result.named = await NameService.resolve([binary.id])
+    return result
+  }
+}
+
+/** NZBs that can be read from a single, unencrypted archive. */
+function hasNzbEntries(info: RarInfo): boolean {
+  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
+}
+
+function isNzbEntry(entry: RarEntry): boolean {
+  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
+}
+
+/** Small posted files (NZBs) that are read completely. */
+const WHOLE_MAX_PARTS = 40
+const WHOLE_MAX_BYTES = 25 * 1024 * 1024
+
+export default new ContentService()