usenet-indexer/app/services/ContentService.ts

import db from '@adonisjs/lucid/services/db'
import logger from '@adonisjs/core/services/logger'
import NntpService from '#services/NntpService'
import Par2Service, { type Par2Result } from '#services/Par2Service'
import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService'
import NameService from '#services/NameService'
import { decodeSegment } from '#lib/yenc/index'
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
import {
  isArchiveStart,
  isLaterVolume,
  NICOVIDEO_FILE,
  NICOVIDEO_RELEASE,
  nicovideoReleaseName,
  type ContentType,
  type NicovideoInfo,
} from '#lib/content/index'
import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe'
import type { Segment } from '#lib/nzb/index'
import { fileStem, releaseIdentity } from '#lib/release/index'
import IngestService from '#services/IngestService'
import { looksLikeFilename } from '#lib/subject/index'
import { sevenZipVolume } from '#lib/sevenzip/index'
import SevenZipService from '#services/SevenZipService'
import SfvService from '#services/SfvService'
import ArchiveWalkService from '#services/ArchiveWalkService'
import { segmentsOf } from '#services/FileStore'

/**
 * MD5 of the first 16 KiB of a file, as par2 records it, from the body of
 * the file's first part. Undefined when that part is shorter than 16 KiB but
 * the file is not (the chunk would span two parts).
 */
export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
  return chunkMd5(decodeSegment(body).data, fileSize)
}

export interface ContentResult {
  par2?: Par2Result
  archive?: ArchiveResult
  named: number
  contentType: ContentType | null
  nzb?: NzbImportResult
  /** NZBs read from inside an archive. */
  archiveNzbs?: NzbImportResult[]
  /** Files an SFV's listing moved into its release. */
  sfvMoved?: number
  /** A niconico capture's release name from its API response (D34). */
  nicovideo?: string
  /** The real name from the yEnc header, for a file posted under a random subject (D36). */
  yencName?: string
}

/**
 * Takes names and groupings from file contents (CONCEPT.md D21). For a
 * completed file with a random name, one article (the first) is enough:
 * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
 * - par2 files are recognised by content and name every file of their set,
 * - RAR headers give the release grouping, name and contents.
 * Inspection is prepare (database) → probe (downloads and parsing,
 * lib/content/probe; here or on a seeding node, D41) → apply (database).
 */
class ContentService {
  /** Random or missing names, and named first RAR volumes (for contents and password flag). */
  isCandidate(name: string) {
    // Even under a random stem: the first volume and the index .par2 are inspected instead.
    if (isLaterVolume(name)) return false
    if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
    if (isArchiveStart(name)) return true
    if (/\.nzb$/i.test(name)) return true
    // SFVs name the files of their release; niconico captures carry their title (D34).
    if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
    // Every 7z volume: the set is listed when its last volume is in (see inspect).
    if (sevenZipVolume(name) !== null) return true
    return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
  }

  /**
   * Uploads that randomise the subject per article often keep the real file
   * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
   * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
   * leaves its weak one-file release for the release its name gives, where
   * the other files of the upload gather. Returns the name, or undefined.
   */
  private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
    const current = String(file.real_name ?? file.filename)
    if (file.real_name || !looksRandom(fileStem(current))) return undefined
    if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined

    const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
    const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
    let releaseId = file.release_id
    if (weak) {
      const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
      if (identity) {
        const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
          { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
        ])
        releaseId = ids.get(identity.key) ?? releaseId
      }
    }
    await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
    if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
    return yencName
  }

  /**
   * Names a niconico capture's release after the video, while it still has
   * the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
   * the search text.
   */
  private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
    if (!file.release_id) return undefined
    const release = await db.from('releases').where('id', file.release_id).first()
    if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
    const name = nicovideoReleaseName(info, id)
    await db
      .from('releases')
      .where('id', release.id)
      .update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
    return name
  }

  /**
   * The database half before a probe (CONCEPT.md D41): what to inspect for
   * a file, or null when there's nothing to do. A later 7z volume becomes its
   * archive's first volume, once all volumes are in.
   */
  async prepare(fileId: number): Promise<InspectTask | null> {
    const file = await db.from('files').where('id', fileId).first()
    if (!file || !this.isCandidate(file.filename)) return null
    if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
      const volumes = await SevenZipService.volumes(file)
      if (!volumes || volumes[0].archive) return null
      return this.prepare(Number(volumes[0].id))
    }
    const segments = segmentsOf(file)
    if (!segments[0]?.id) return null
    // A 7z listing reads the last volume's tail (sniffing decides whether it's 7z: random names too).
    const volumes = await SevenZipService.volumes(file)
    const last = volumes && volumes.length > 1 ? segmentsOf(volumes[volumes.length - 1]) : segments
    return {
      kind: 'inspect',
      fileId: Number(file.id),
      name: String(file.real_name ?? file.filename),
      segments: taskSegments(segments),
      sevenZip: volumes ? { volumes: volumes.length, last: taskSegments(last, true) } : null,
    }
  }

  /** The database half after a probe: names, groupings, listings and flags from what it found. */
  async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
    const result: ContentResult = { named: 0, contentType: probe.contentType }
    const file = await db.from('files').where('id', task.fileId).first()
    if (!file) return result
    const binary = await db.from('binaries').where('file_id', task.fileId).first()
    const { contentType } = probe
    await db.from('files').where('id', file.id).update({ content_type: contentType })

    // Fingerprint, so par2 sets (now or later) can name this file exactly.
    if (binary) {
      await db
        .from('binaries')
        .where('id', binary.id)
        .update({
          size: db.raw('coalesce(size, ?)', [probe.size]),
          md5_16k: db.raw('coalesce(md5_16k, ?)', [probe.md5_16k]),
          // Inspected: the GC may delete it again by its usual rules (LOG.md 2026-10-06 (11)).
          awaiting_inspection: null,
        })
    }

    result.yencName = await this.nameFromYenc(file, binary, probe.yencName ?? '')
    if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
      // An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
      // to read, and useless without the key.
      const releaseId = file.release_id ?? binary?.release_id
      if (releaseId) {
        await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
      }
      return result
    }
    const nicovideo = NICOVIDEO_FILE.exec(task.name)
    if (nicovideo) {
      if (probe.nicovideo) result.nicovideo = await this.nameNicovideo(file, probe.nicovideo, nicovideo[1])
      return result
    }
    if (/\.sfv$/i.test(task.name)) {
      if (typeof probe.text === 'string') result.sfvMoved = (await SfvService.attach(file, probe.text)) ?? undefined
      return result
    }
    if (contentType === 'nzb') {
      if (typeof probe.text === 'string') result.nzb = (await NzbImportService.import(file, probe.text)) ?? undefined
      return result
    }
    if (probe.par2 && binary) {
      result.par2 = await Par2Service.attach(binary, probe.par2)
    } else if (probe.archive && probe.archive.format === '7z') {
      result.archive = await ArchiveService.record(file, binary, probe.archive)
    } else if (probe.archive) {
      result.archive = await ArchiveService.record(file, binary, probe.archive)
      // The rest of the listing, later and at low priority (D40).
      await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, probe.archive)
      if (probe.archiveNzbs?.length) {
        result.archiveNzbs = []
        for (const { entryName, xml } of probe.archiveNzbs) {
          const imported = await NzbImportService.import(file, xml, { entryName, moveContainer: (probe.archiveNzbEntries ?? 0) === 1 })
          if (imported) result.archiveNzbs.push(imported)
        }
      }
    }
    if (binary) result.named = await NameService.resolve([binary.id])
    return result
  }

  /** Inspects a file here: prepare, probe on the inspection pool's connections, apply. */
  async inspect(fileId: number): Promise<ContentResult | null> {
    const task = await this.prepare(fileId)
    if (!task) return null
    const probe = await probeInspection(task, fetchPooled, (message) => logger.warn(message))
    return this.apply(task, probe)
  }
}

/** A task's segment list: all of a small file's; of a big one, the first and the last few (a 7z tail). */
function taskSegments(segments: Segment[], tailOnly = false): TaskSegment[] {
  const list = segments.map(({ number, id, size }) => ({ number, id, size }))
  if (list.length <= TASK_SEGMENTS) return list
  return tailOnly ? list.slice(-TASK_SEGMENTS) : [list[0], ...list.slice(-TASK_SEGMENTS)]
}

/** More than whole files (40 parts) and par2 files (20) are read; a 7z tail of 16 MB fits easily. */
const TASK_SEGMENTS = 40

/** One article body on a pooled connection. */
async function fetchPooled(messageId: string): Promise<Buffer> {
  const conn = await NntpService.acquire()
  let failed = false
  try {
    return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
  } catch (error: any) {
    // A missing article (430) leaves the connection fine.
    failed = !/^4\d\d/.test(String(error.message))
    throw error
  } finally {
    NntpService.release(conn, failed)
  }
}

/**
 * One line per finding, as worker:collection logs them (and the seed API
 * for nodes' results).
 */
export function describeContent(filename: string, content: ContentResult | null): string[] {
  const lines: string[] = []
  if (content?.par2) {
    const { par2 } = content
    lines.push(`Par2 "${filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
  } else if (content?.archive) {
    const { info, releaseRenamed, grouped } = content.archive
    const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
      .filter(Boolean)
      .join(', ')
    lines.push(
      `Archive "${filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
        (grouped ? ', grouped into a release' : '') +
        (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
    )
  }
  if (content?.named) lines.push(`"${filename}" got its real name from par2.`)
  if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
  if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
  if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
  if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
  if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
  if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
  if (content?.nzb) {
    const n = content.nzb
    lines.push(`NZB "${filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
  }
  return lines
}

export default new ContentService()