usenet-indexer/lib/ingest/segment.ts

import { parseXrefGroups } from '../nntp/xref.js'
import { fileKey, parseSubject, weakFileKey } from '../subject/index.js'
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'
import { inferTotalParts, type YencMeta } from '../yenc/index.js'

/** One segment, with the identity of the file it belongs to already resolved. */
export interface IngestSegment {
  fileKey: string
  poster: string
  /** Subject without segment counter. */
  subject: string
  name: string
  totalParts: number
  number: number
  /** Without angle brackets. */
  messageId: string
  bytes: number
  articleNumber?: bigint | number
  postedAt?: Date
  groups: string[]
  /**
   * The identity comes from the yEnc name only (obfuscated posts). Such names
   * can be shared by several equal-sized files of one post, so a second
   * article for the same part number marks the binary as conflicted.
   */
  weakIdentity?: boolean
  /** Release (download) this file belongs to, when it can be determined. */
  release?: ReleaseIdentity | null
  /** Exact file size from =ybegin (body-derived segments only). */
  size?: number
  /** yEnc CRCs (body-derived segments only), to verify split volumes. */
  pcrc32?: string
  partSize?: number
  fileCrc32?: string
  /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
  md5_16k?: string
}

/** An XOVER overview entry as returned by nntp-js. */
export type OverviewHeader = Record<string, string>

function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
  const xrefGroups = parseXrefGroups(header.xref)
  return {
    poster: header.from ?? '',
    messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
    bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
    articleNumber,
    postedAt: header.date ? new Date(header.date) : undefined,
    groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
  }
}

/** Builds a segment from the subject line; null when the subject is obfuscated. */
export function segmentFromHeader(
  header: OverviewHeader,
  groupName: string,
  articleNumber?: bigint | number
): IngestSegment | null {
  const parsed = header.subject ? parseSubject(header.subject) : null
  if (!parsed || !header['message-id']) return null
  const base = common(header, groupName, articleNumber)
  return {
    ...base,
    fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
    subject: parsed.base,
    name: parsed.name,
    totalParts: parsed.totalSegments,
    number: parsed.segment,
    release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
  }
}

/**
 * Builds a segment for an obfuscated post from the yEnc header in its body.
 * Subject and poster can be random per article, so the identity comes from
 * the yEnc header only (see weakFileKey).
 */
export function segmentFromYenc(
  header: OverviewHeader,
  groupName: string,
  meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
  articleNumber?: bigint | number
): IngestSegment | null {
  const base = common(header, groupName, articleNumber)
  const totalParts = inferTotalParts(meta)
  const number = meta.begin.part ?? 1
  if (!base.messageId || number < 1 || number > totalParts) return null
  const subject = `"${meta.begin.name}" yEnc`
  return {
    ...base,
    fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
    subject,
    name: meta.begin.name,
    totalParts,
    number,
    weakIdentity: true,
    release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
    size: meta.begin.size,
    // Single-part posts only carry crc32, which is then also the part's CRC.
    pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
    partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
    fileCrc32: meta.end?.crc32,
  }
}