usenet-indexer/app/services/ContentService.ts

import { createHash } from 'node:crypto'
import db from '@adonisjs/lucid/services/db'
import NntpService from '#services/NntpService'
import Par2Service, { type Par2Result } from '#services/Par2Service'
import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
import NameService from '#services/NameService'
import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
import {
  isArchiveStart,
  isLaterVolume,
  NICOVIDEO_FILE,
  NICOVIDEO_RELEASE,
  nicovideoInfo,
  nicovideoReleaseName,
  sniffContentType,
  zstdPrefix,
  type ContentType,
  type NicovideoInfo,
} from '#lib/content/index'
import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
import { fileStem, releaseIdentity } from '#lib/release/index'
import IngestService from '#services/IngestService'
import { looksLikeFilename } from '#lib/subject/index'
import { sevenZipVolume } from '#lib/sevenzip/index'
import SevenZipService from '#services/SevenZipService'
import SfvService from '#services/SfvService'
import { segmentsOf } from '#services/FileStore'

const FIRST_CHUNK = 16 * 1024

/**
 * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
 * the file's first part. Undefined when that part is shorter than 16 KiB but
 * the file is not (the chunk would span two parts).
 */
export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
  return chunkMd5(decodeSegment(body).data, fileSize)
}

function chunkMd5(data: Buffer, fileSize: number): string | undefined {
  if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
  return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
}

export interface ContentResult {
  par2?: Par2Result
  archive?: ArchiveResult
  named: number
  contentType: ContentType | null
  nzb?: NzbImportResult
  /** NZBs read from inside an archive. */
  archiveNzbs?: NzbImportResult[]
  /** Files an SFV's listing moved into its release. */
  sfvMoved?: number
  /** A niconico capture's release name from its API response (D34). */
  nicovideo?: string
  /** The real name from the yEnc header, for a file posted under a random subject (D36). */
  yencName?: string
}

/**
 * Takes names and groupings from file contents (CONCEPT.md D21). For a
 * completed file with a random name, one article (the first) is enough:
 * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
 * - par2 files are recognised by content and name every file of their set,
 * - RAR headers give the release grouping, name and contents.
 */
class ContentService {
  /** Random or missing names, and named first RAR volumes (for contents and password flag). */
  isCandidate(name: string) {
    // Even under a random stem: the first volume and the index .par2 are inspected instead.
    if (isLaterVolume(name)) return false
    if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
    if (isArchiveStart(name)) return true
    if (/\.nzb$/i.test(name)) return true
    // SFVs name the files of their release; niconico captures carry their title (D34).
    if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
    // Every 7z volume: the set is listed when its last volume is in (see inspect).
    if (sevenZipVolume(name) !== null) return true
    return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
  }

  /**
   * Uploads that randomise the subject per article often keep the real file
   * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
   * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
   * leaves its weak one-file release for the release its name gives, where
   * the other files of the upload gather. Returns the name, or undefined.
   */
  private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
    const current = String(file.real_name ?? file.filename)
    if (file.real_name || !looksRandom(fileStem(current))) return undefined
    if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined

    const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
    const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
    let releaseId = file.release_id
    if (weak) {
      const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
      if (identity) {
        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
          { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
        ])
        releaseId = ids.get(identity.key) ?? releaseId
      }
    }
    await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
    if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
    return yencName
  }

  /**
   * Names a niconico capture's release after the video, while it still has
   * the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
   * the search text.
   */
  private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
    if (!file.release_id) return undefined
    const release = await db.from('releases').where('id', file.release_id).first()
    if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
    const name = nicovideoReleaseName(info, id)
    await db
      .from('releases')
      .where('id', release.id)
      .update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
    return name
  }

  /** Downloads a whole (small) file whose first segment is already decoded. */
  private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
    const parts = segmentsOf(file)
    const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
    if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
    if (parts.length === 1) return first

    let assembled: YencFile | undefined
    const conn = await NntpService.acquire()
    let failed = false
    try {
      for (const part of parts) {
        const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
        assembled ??= YencFile.fromSegment(segment)
        assembled.add(segment)
      }
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
    return assembled!.getBuffer()
  }

  /**
   * Downloads a small archive that lists NZBs and imports each of them
   * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
   * uploads by other posters.
   */
  private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
    const archive = await this.readWhole(first, file)
    const info = archive && parseRarHeaders(archive)
    if (!archive || !info) return []
    const entries = info.entries.filter(isNzbEntry)
    const results: NzbImportResult[] = []
    for (const entry of entries) {
      const xml = await extractEntry(archive, entry)
      if (!xml) continue
      const imported = await NzbImportService.import(file, xml.toString('utf8'), {
        entryName: entry.name,
        moveContainer: entries.length === 1,
      })
      if (imported) results.push(imported)
    }
    return results
  }

  async inspect(fileId: number): Promise<ContentResult | null> {
    const file = await db.from('files').where('id', fileId).first()
    const binary = await db.from('binaries').where('file_id', fileId).first()
    if (!file || !this.isCandidate(file.filename)) return null
    // A later 7z volume: list the set from its first volume once all volumes are in.
    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
      const volumes = await SevenZipService.volumes(file)
      if (!volumes || volumes[0].archive) return null
      return this.inspect(Number(volumes[0].id))
    }
    const first = segmentsOf(file)[0]
    if (!first?.id) return null

    const conn = await NntpService.acquire()
    let failed = false
    let body: Buffer
    try {
      body = await NntpService.fetchBody(conn, first.id)
    } catch (error) {
      failed = true
      throw error
    } finally {
      NntpService.release(conn, failed)
    }
    // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
    let segment
    try {
      segment = decodeSegment(body)
    } catch (error) {
      if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
      segment = decodeSegment(body, { lenientSize: true })
    }
    const size = segment.meta.begin.size
    const contentType = sniffContentType(segment.data)
    await db.from('files').where('id', fileId).update({ content_type: contentType })

    // Fingerprint, so par2 sets (now or later) can name this file exactly.
    if (binary) {
      await db
        .from('binaries')
        .where('id', binary.id)
        .update({
          size: db.raw('coalesce(size, ?)', [size]),
          md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
        })
    }

    const result: ContentResult = { named: 0, contentType }
    result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
    if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
      // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
      // to read, and useless without the key.
      const releaseId = file.release_id ?? binary?.release_id
      if (releaseId) {
        await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
      }
      return result
    }
    const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
    if (nicovideo) {
      const info = nicovideoInfo(await zstdPrefix(segment.data))
      if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
      return result
    }
    if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
      const text = await this.readWhole(segment.data, file)
      if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
      return result
    }
    if (contentType === 'nzb') {
      const xml = await this.readWhole(segment.data, file)
      if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
      return result
    }
    const set = await Par2Service.read(segment.data, segmentsOf(file))
    if (set && binary) {
      result.par2 = await Par2Service.attach(binary, set)
    } else if (contentType === '7z') {
      const info = await SevenZipService.list(file, segment.data)
      if (info) result.archive = await ArchiveService.record(file, binary, info)
    } else {
      const info = parseRarHeaders(segment.data)
      if (info) {
        result.archive = await ArchiveService.record(file, binary, info)
        if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
      }
    }
    if (binary) result.named = await NameService.resolve([binary.id])
    return result
  }
}

/** NZBs that can be read from a single, unencrypted archive. */
function hasNzbEntries(info: RarInfo): boolean {
  return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
}

function isNzbEntry(entry: RarEntry): boolean {
  return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
}

/** Small posted files (NZBs) that are read completely. */
const WHOLE_MAX_PARTS = 40
const WHOLE_MAX_BYTES = 25 * 1024 * 1024

export default new ContentService()