usenet-indexer/lib/split/instances.ts

import { combineCrc32 } from '../yenc/decode.js'

export interface SplitArticle {
  number: number
  messageId: string
  bytes: number
  /** Article number in the group; posting order. */
  articleNumber: number | null
  /** yEnc part CRC and decoded part length, for verification. */
  pcrc32?: string | null
  partSize?: number | null
  /** Whole-file CRC32, usually only in the last part. */
  fileCrc32?: string | null
}

export interface SplitOptions {
  /** Total parts per volume; a volume that has them all takes no more articles. */
  totalParts: number
  /**
   * Maximum distance in article numbers between consecutive articles of one
   * volume. Separates different posts that happen to share a size.
   */
  window?: number
  /**
   * How far (in posting positions) an article may be from where its volume
   * predicts it; covers out-of-order finishing across connections.
   */
  tolerance?: number
}

interface Volume {
  articles: SplitArticle[]
  /** Posting rank of each part number present. */
  ranks: Map<number, number>
  lastArticle: number
  opened: number
}

/** ~1 minute of posting in a busy group. */
const DEFAULT_WINDOW = 20_000

/**
 * Separates the articles of several equal-sized files that share one
 * identity (obfuscated volumes with random names, possibly from different
 * posts) into one list per file. See CONCEPT.md (D20).
 *
 * Posting tools upload a queue (volume 1 parts 1..n, volume 2 parts 1..n,
 * ...) over several connections, so within a volume part k+1 lands about one
 * position after part k in posting order, give or take the connection count.
 * Each article, in posting order, goes to the eligible volume (lacks this
 * part, not complete, active within `window` article numbers) whose
 * predicted position for it is closest: taken from that volume's article with
 * the nearest part number, which keeps the prediction local and unaffected
 * by missing articles elsewhere. A deviation above the tolerance starts a new
 * volume. The result is recomputed from all articles, so it improves as late
 * articles arrive.
 */
export function splitInstances(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
  const window = options.window ?? DEFAULT_WINDOW
  const tolerance = options.tolerance ?? Math.max(16, Math.ceil(options.totalParts * 0.1))
  const seen = new Set<string>()
  const ordered = articles
    .filter((a) => (seen.has(a.messageId) ? false : (seen.add(a.messageId), true)))
    .sort(
      (a, b) =>
        (a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
        a.messageId.localeCompare(b.messageId)
    )

  const volumes: Volume[] = []
  ordered.forEach((article, rank) => {
    const position = article.articleNumber ?? Number.MAX_SAFE_INTEGER
    let best: Volume | undefined
    let bestScore: [number, number] | undefined
    for (const volume of volumes) {
      if (volume.ranks.has(article.number) || volume.ranks.size >= options.totalParts) continue
      if (position - volume.lastArticle > window) continue
      const deviation = Math.abs(rank - predictedRank(volume, article.number))
      if (deviation > tolerance) continue
      const score: [number, number] = [deviation, volume.opened]
      if (!bestScore || score[0] < bestScore[0] || (score[0] === bestScore[0] && score[1] < bestScore[1])) {
        best = volume
        bestScore = score
      }
    }
    if (!best) {
      best = { articles: [], ranks: new Map(), lastArticle: position, opened: volumes.length }
      volumes.push(best)
    }
    best.articles.push(article)
    best.ranks.set(article.number, rank)
    best.lastArticle = Math.max(best.lastArticle, position)
  })

  return volumes.map((v) => v.articles.sort((a, b) => a.number - b.number))
}

/** Where part `number` should be in posting order, judged from the volume's nearest part. */
function predictedRank(volume: Volume, number: number): number {
  let anchorPart = 0
  let anchorRank = 0
  let nearest = Number.MAX_SAFE_INTEGER
  for (const [part, rank] of volume.ranks) {
    const distance = Math.abs(part - number)
    if (distance < nearest) {
      nearest = distance
      anchorPart = part
      anchorRank = rank
    }
  }
  return anchorRank + (number - anchorPart)
}

/**
 * Simpler split: for every part number, the i-th article in posting order
 * belongs to volume i. Right for a single, complete post; used as a second
 * proposal for verification.
 */
export function splitByRank(articles: SplitArticle[]): SplitArticle[][] {
  const byNumber = new Map<number, SplitArticle[]>()
  const seen = new Set<string>()
  for (const article of articles) {
    if (seen.has(article.messageId)) continue
    seen.add(article.messageId)
    byNumber.set(article.number, [...(byNumber.get(article.number) ?? []), article])
  }
  const volumes: SplitArticle[][] = []
  for (const group of byNumber.values()) {
    group.sort(
      (a, b) =>
        (a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) ||
        a.messageId.localeCompare(b.messageId)
    )
    group.forEach((article, i) => (volumes[i] ??= []).push(article))
  }
  return volumes.map((v) => v.sort((a, b) => a.number - b.number))
}

/**
 * True when a volume has every part and its combined part CRCs equal the
 * whole-file CRC32 its last part announces: then it is exactly one file.
 */
export function verifyVolume(volume: SplitArticle[], totalParts: number): boolean {
  if (volume.length !== totalParts) return false
  const sorted = [...volume].sort((a, b) => a.number - b.number)
  if (sorted.some((a, i) => a.number !== i + 1 || !a.pcrc32 || !a.partSize)) return false
  const expected = sorted[sorted.length - 1].fileCrc32
  if (!expected) return false
  const combined = combineCrc32(sorted.map((a) => ({ crc: a.pcrc32!, length: a.partSize! })))
  return combined === expected.toLowerCase().padStart(8, '0')
}

/**
 * Splits and keeps only volumes that verify (verifyVolume). Proposals from
 * both methods are tried; articles of verified volumes are taken out and the
 * rest is tried again, until nothing new verifies. Unverified articles are
 * left out: guessing wrong would produce a corrupt file.
 */
export function splitVerified(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] {
  const verified: SplitArticle[][] = []
  let remaining = articles
  for (let round = 0; round < 4 && remaining.length >= options.totalParts; round++) {
    const found: SplitArticle[][] = []
    const used = new Set<string>()
    for (const propose of [(a: SplitArticle[]) => splitInstances(a, options), splitByRank]) {
      for (const volume of propose(remaining.filter((a) => !used.has(a.messageId)))) {
        if (verifyVolume(volume, options.totalParts)) {
          found.push(volume)
          for (const article of volume) used.add(article.messageId)
        }
      }
    }
    if (found.length === 0) break
    verified.push(...found)
    remaining = remaining.filter((a) => !used.has(a.messageId))
  }
  return verified
}