usenet-indexer/lib/probe/inspect.ts

import { createHash } from 'node:crypto'
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
import { isPar2, parsePar2 } from '#lib/par2/index'
import { parseRarHeaders } from '#lib/rar/index'
import { parseSevenZipTail } from '#lib/sevenzip/index'
import { parseZipStart } from '#lib/zip/index'
import { sniffContentType, type ContentType } from '#lib/content/index'
import { fileStem, looksRandom } from '#lib/release/index'

/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
export interface Inspection {
  /** The file name in the yEnc header. */
  yencName: string
  /** The whole file's size, from the yEnc header. */
  size: number
  type: ContentType | null
  /** Names inside: an archive's entries or a par2 set's files, the largest first, at most MAX_NAMES. */
  names?: string[]
  /** How many entries or files there are in all. */
  entries?: number
  /** An archive with encrypted entries or headers. */
  passworded?: boolean
  /**
   * MD5 of the file's first 16 KiB (hex), when this article starts the file: what a par2 set's FileDesc
   * holds per file, so a renamed file is recognised by it (the par2 merge; app/services/par2_merger.ts).
   */
  hash16k?: string
  /** A par2 body's recovery set: every file it describes (not kept in release_inspections; par2_files). */
  par2?: {
    setId: string
    files: { fileId: string; hash16k: string; length: number; name: string }[]
  }
}

const HASH_BYTES = 16 * 1024

/** The inspection's version: raised when inspect() changes, so probe_cache rows of an older one are read again. */
export const PROBE_VERSION = 2

/** Names kept per inspection: the catalog wants the content's name, not every volume's. */
export const MAX_NAMES = 8
const MAX_NAME_LENGTH = 200

/** Archive volumes, par2 volumes and split parts: names of the packaging, not the content. */
const PACKAGING = /\.(rar|r\d{2,3}|s\d{2}|\d{3}|7z(\.\d+)?|zip|z\d{2}|par2|sfv|nfo|nzb)$/i

/** Decodes a body; some tools declare a size that leaves out a header, the CRC still has to match. */
function decodeBody(body: Buffer): DecodedSegment {
  try {
    return decodeSegment(body)
  } catch (error) {
    if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
    return decodeSegment(body, { lenientSize: true })
  }
}

function largestFirst(items: { name: string; size: number }[]): string[] {
  return items
    .sort((a, b) => b.size - a.size)
    .slice(0, MAX_NAMES)
    .map((i) => i.name.slice(0, MAX_NAME_LENGTH))
}

/**
 * One article's body: its yEnc header, what the first bytes are, and for par2, RAR, ZIP and 7z what's inside.
 * Throws when it isn't yEnc. `kind` is the probe candidate's: a 7z-tail's article ends the archive.
 */
export function inspect(body: Buffer, kind?: string): Inspection {
  const segment = decodeBody(body)
  const result: Inspection = {
    yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH),
    size: segment.meta.begin.size,
    type: sniffContentType(segment.data),
  }
  // The file's first 16 KiB, or all of a smaller file.
  if (
    segment.offset === 0 &&
    (segment.data.length >= HASH_BYTES || segment.data.length === result.size)
  ) {
    result.hash16k = createHash('md5').update(segment.data.subarray(0, HASH_BYTES)).digest('hex')
  }
  if (isPar2(segment.data)) {
    // An index file fits one article; what a larger one cut off is skipped by the parser.
    const set = parsePar2(segment.data)
    if (set) {
      result.entries = set.files.length
      result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
      result.par2 = {
        setId: set.recoverySetId,
        files: set.files.map((f) => ({
          fileId: f.fileId,
          hash16k: f.md5_16k,
          length: f.length,
          name: f.name.slice(0, MAX_NAME_LENGTH),
        })),
      }
    }
    return result
  }
  if (result.type === 'rar4' || result.type === 'rar5') {
    const info = parseRarHeaders(segment.data)
    if (info) {
      const files = info.entries.filter((e) => !e.directory)
      result.entries = files.length
      result.names = largestFirst(files.map((e) => ({ name: e.name, size: Number(e.size ?? 0) })))
      result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
    }
  }
  if (result.type === 'zip') {
    const start = parseZipStart(segment.data)
    if (start) {
      result.entries = start.files.length
      result.names = largestFirst(start.files)
      result.passworded = start.encrypted
    }
  }
  // 7z keeps its listing at the end: the last segment of the last volume (a 7z-tail), or a small archive whole.
  if (
    (kind === '7z-tail' || result.type === '7z') &&
    segment.offset + segment.data.length === result.size
  ) {
    const listing = parseSevenZipTail(segment.data)
    if (listing) {
      result.entries = listing.files.length
      if (listing.files.length > 0) result.names = largestFirst(listing.files)
      result.passworded = listing.encrypted
    }
  }
  return result
}

/**
 * The usable name one probe of an unnamed release found (the pending ladder, REBUILD.md decision f): the largest
 * named file of a par2 set or an archive's listing, or a lone file's yEnc name, when it isn't packaging and isn't
 * random itself (an obfuscated upload's par2 names its random volumes; its yEnc name is mostly the random name
 * again). Null: this probe named nothing, the next target is tried.
 */
export function usableName(inspection: Inspection, kind: string): string | null {
  const usable = (n: string | undefined) => {
    if (!n) return null
    const name = n.replace(/^.*[\\/]/, '').slice(0, 255)
    return name && !PACKAGING.test(name) && !looksRandom(fileStem(name)) ? name : null
  }
  const named = (inspection.names ?? []).map(usable).find((n) => n)
  if (named) return named
  return kind === 'lone' || !inspection.names?.length ? usable(inspection.yencName) : null
}

/**
 * A release's best inner name from its inspections (by probe kind): the
 * largest named entry of an archive, else the largest file of a par2 set
 * that isn't packaging, else a lone file's yEnc name. Null when none says
 * more than the release's own name.
 */
export function innerName(
  inspections: { kind: string; result: Inspection }[],
  releaseName = ''
): string | null {
  const of = (kind: string) => inspections.filter((i) => i.kind === kind).map((i) => i.result)
  const content = (kind: string) =>
    of(kind)
      .flatMap((i) => i.names ?? [])
      .find((n) => !PACKAGING.test(n))
  const pick =
    content('archive') ??
    content('7z-tail') ??
    content('par2') ??
    of('lone').find((i) => i.yencName && !PACKAGING.test(i.yencName))?.yencName
  const name = pick?.replace(/^.*[\\/]/, '').slice(0, 255)
  // The release's own name with an extension says nothing new (a random-named file's yEnc name is mostly its random name).
  return name && !name.toLowerCase().startsWith(releaseName.toLowerCase() || '\0') ? name : null
}