usenet-indexer/lib/edge/range_indexer.ts

import { createHash } from 'node:crypto'
import type { Overview } from '../nntp/index.js'
import { parseXrefGroups } from '../nntp/index.js'
import { segmentFromHeader, type IngestSegment } from '../headers/index.js'
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
import { RuleSet, isRandomFile, ownerOf, type Rules } from '../rules/index.js'
import { encodeChunk, type ChunkFile } from '../chunks/index.js'

/** A release's files from one batch (REBUILD.md, chunk store): what the catalog adds up, and the encoded chunk. */
export interface ReleaseChunk {
  key: string
  name: string
  source: string
  fileTotal: number | null
  poster: string
  groups: string[]
  postedAt: number | null
  files: number
  segments: number
  bytes: number
  firstArticle: number
  lastArticle: number
  /** A file with a readable name; a par2 file. */
  readable: boolean
  hasPar2: boolean
  payload: Buffer
}

export interface DroppedChunk extends ReleaseChunk {
  rule: string
  ruleKey: string
}

export type CandidateKind = 'archive' | 'par2' | 'lone' | '7z-tail'

/** An article phase 2 should read (REBUILD.md, probe queue). */
export interface ProbeCandidate {
  releaseKey: string
  kind: CandidateKind
  messageId: string
  file: string
}

/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
export interface RuleCount {
  rule: 'crosspost' | 'obfuscated' | 'garbage' | 'per-article' | 'incomplete'
  key: string
  articles: number
  files: number
}

/** A segment of a file cut off by the range's edges, for the neighbouring range. */
export interface Fragment {
  fileKey: string
  name: string
  poster: string
  subject: string
  totalParts: number
  groups: string[]
  release: { key: string; name: string; source: string; fileTotal?: number } | null
  number: number
  messageId: string
  bytes: number
  articleNumber: number
  postedAt: number | null
}

export interface EdgeBatch {
  /** The range's resume point this batch starts at. */
  from: number
  /** First article number not covered yet. */
  next: number
  headers: number
  releases: ReleaseChunk[]
  samples: DroppedChunk[]
  candidates: ProbeCandidate[]
  counts: RuleCount[]
  fragments: Fragment[]
}

export interface RangeIndexerOptions {
  group: string
  /** Inclusive article numbers. */
  start: number
  end: number
  rules: Rules
  /** Chunks without a new part before an open file is given up (as a fragment near an edge). */
  idleChunks?: number
  /** Within this many articles of an edge, a cut-off file is kept as fragments. */
  edgeArticles?: number
}

const MAX_CANDIDATES = 4

/**
 * One leased range on an edge worker (REBUILD.md, part 2): overview lines
 * in, chunk by chunk and in order; releases, probe candidates, rule counts
 * and edge fragments out, per batch (`take`). Holds only the files still
 * waiting for parts.
 */
export class RangeIndexer {
  private aggregator = new FileAggregator<IngestSegment>()
  private rules: RuleSet
  private batch: EdgeBatch
  private counts = new Map<string, RuleCount>()
  private readonly idle: number
  private readonly edge: number

  constructor(private options: RangeIndexerOptions) {
    this.rules = new RuleSet(options.rules)
    this.idle = options.idleChunks ?? 5
    this.edge = options.edgeArticles ?? 50_000
    this.batch = this.empty(options.start)
  }

  /** One chunk of overview lines; `next` is the first article after it. */
  add(overviews: Overview[], next: number) {
    const { group, rules } = this.options
    const segments: IngestSegment[] = []
    for (const [articleNumber, header] of overviews) {
      if (!Number.isFinite(articleNumber) || !header['message-id']) continue
      this.batch.headers++
      const groups = parseXrefGroups(header.xref)
      const owner = ownerOf(groups.length > 0 ? groups : [group], rules.priorities) ?? group
      if (owner !== group) {
        this.count('crosspost', owner, 1, 0)
        continue
      }
      const segment = segmentFromHeader(header, group, articleNumber)
      if (segment) segments.push(segment)
      else this.count('obfuscated', '', 1, 0)
    }
    this.files(this.aggregator.add(segments))
    this.cutOff(this.aggregator.evict(this.idle))
    this.batch.next = next
  }

  /** The end of the range: every open file is given up. */
  finish() {
    this.cutOff(this.aggregator.evict(0))
    this.batch.next = this.options.end + 1
  }

  /** What was found since the last take, and a fresh batch. */
  take(): EdgeBatch {
    this.judgeReleases()
    const out = this.batch
    out.counts = [...this.counts.values()]
    this.counts.clear()
    this.batch = this.empty(out.next)
    return out
  }

  get openFiles() {
    return this.aggregator.openFiles
  }

  /** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
  private completed = new Map<string, AggregatedFile<IngestSegment>[]>()

  private files(files: AggregatedFile<IngestSegment>[]) {
    for (const file of files) {
      const key = releaseKeyOf(file.segments[0])
      const list = this.completed.get(key)
      if (list) list.push(file)
      else this.completed.set(key, [file])
    }
  }

  /** Judges and encodes the releases completed in this batch. */
  private judgeReleases() {
    const byRelease = this.completed
    this.completed = new Map()
    for (const [key, list] of byRelease) {
      const verdict = this.rules.judge(
        key,
        list.map((f) => ({ name: f.segments[0].name, totalParts: f.segments[0].totalParts, groups: f.segments[0].groups, firstMessageId: f.segments[0].messageId }))
      )
      if (verdict.rule === null) {
        this.batch.releases.push(chunkOf(key, list))
        this.batch.candidates.push(...candidatesOf(key, list))
        continue
      }
      this.count(verdict.rule, verdict.key, list.reduce((n, f) => n + f.segments.length, 0), list.length)
      if (verdict.sample) this.batch.samples.push({ ...chunkOf(key, list), rule: verdict.rule, ruleKey: verdict.key })
    }
  }

  /** Given-up segments: kept near the range's edges (the neighbour may complete them), counted otherwise. */
  private cutOff(segments: IngestSegment[]) {
    const { start, end } = this.options
    for (const s of segments) {
      const n = Number(s.articleNumber)
      if (Number.isFinite(n) && (n < start + this.edge || n > end - this.edge)) this.batch.fragments.push(fragmentOf(s))
      else this.count('incomplete', '', 1, 0)
    }
  }

  private count(rule: RuleCount['rule'], key: string, articles: number, files: number) {
    const id = `${rule}\n${key}`
    const c = this.counts.get(id)
    if (c) {
      c.articles += articles
      c.files += files
    } else {
      this.counts.set(id, { rule, key, articles, files })
    }
  }

  private empty(from: number): EdgeBatch {
    return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
  }
}

/** The release key; a file without a recognisable release becomes a release of its own. */
function releaseKeyOf(s: IngestSegment): string {
  return s.release?.key ?? createHash('sha1').update(`file\n${s.fileKey}`).digest('hex')
}

function postedOf(segments: IngestSegment[]): number | null {
  let min: number | null = null
  for (const s of segments) {
    const t = s.postedAt?.getTime()
    if (t !== undefined && Number.isFinite(t) && (min === null || t < min)) min = t
  }
  return min
}

function chunkOf(key: string, files: AggregatedFile<IngestSegment>[]): ReleaseChunk {
  const first = files[0].segments[0]
  const chunkFiles: ChunkFile[] = files.map((f) => {
    const s = f.segments[0]
    return { name: s.name, poster: s.poster, subject: s.subject, postedAt: postedOf(f.segments), groups: s.groups, segments: f.segments.map((x) => [x.number, x.messageId, x.bytes]) }
  })
  let segments = 0
  let bytes = 0
  let firstArticle = Number.POSITIVE_INFINITY
  let lastArticle = 0
  let postedAt: number | null = null
  for (const f of files) {
    for (const s of f.segments) {
      segments++
      bytes += s.bytes
      const n = Number(s.articleNumber)
      if (n < firstArticle) firstArticle = n
      if (n > lastArticle) lastArticle = n
    }
    const t = postedOf(f.segments)
    if (t !== null && (postedAt === null || t < postedAt)) postedAt = t
  }
  return {
    key,
    name: first.release?.name ?? first.name,
    source: first.release?.source ?? 'file',
    fileTotal: files.find((f) => f.segments[0].release?.fileTotal)?.segments[0].release?.fileTotal ?? null,
    poster: first.poster,
    groups: first.groups,
    postedAt,
    files: files.length,
    segments,
    bytes,
    firstArticle,
    lastArticle,
    readable: chunkFiles.some((f) => !isRandomFile(f.name)),
    hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
    payload: encodeChunk(chunkFiles),
  }
}

/**
 * The articles that say what a release is: the first of its first archive
 * volume, of its par2 index, of a lone random-named file, and a 7z set's
 * last volume (its listing is in the tail). At most a few per release.
 */
function candidatesOf(key: string, files: AggregatedFile<IngestSegment>[]): ProbeCandidate[] {
  const out: ProbeCandidate[] = []
  let tail: { volume: number; file: AggregatedFile<IngestSegment> } | null = null
  for (const f of files) {
    const name = f.segments[0].name
    const volume = sevenZipVolume(name)
    if (volume !== null && volume > 1) {
      if (!tail || volume > tail.volume) tail = { volume, file: f }
      continue
    }
    if (isArchiveStart(name)) out.push({ releaseKey: key, kind: 'archive', messageId: f.segments[0].messageId, file: name })
    else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, kind: 'par2', messageId: f.segments[0].messageId, file: name })
  }
  if (tail) out.push({ releaseKey: key, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].name })
  if (out.length === 0 && files.length === 1 && isRandomFile(files[0].segments[0].name)) {
    out.push({ releaseKey: key, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].name })
  }
  return out.slice(0, MAX_CANDIDATES)
}

function fragmentOf(s: IngestSegment): Fragment {
  return {
    fileKey: s.fileKey,
    name: s.name,
    poster: s.poster,
    subject: s.subject,
    totalParts: s.totalParts,
    groups: s.groups,
    release: s.release ? { key: s.release.key, name: s.release.name, source: s.release.source, ...(s.release.fileTotal ? { fileTotal: s.release.fileTotal } : {}) } : null,
    number: s.number,
    messageId: s.messageId,
    bytes: s.bytes,
    articleNumber: Number(s.articleNumber),
    postedAt: s.postedAt && Number.isFinite(s.postedAt.getTime()) ? s.postedAt.getTime() : null,
  }
}