usenet-indexer/lib/edge/range_indexer.ts

import { createHash } from 'node:crypto'
import type { Overview } from '../nntp/index.js'
import { FileAggregator, type AggregatedFile, isArchiveStart, isLaterVolume, sevenZipVolume } from '../files/index.js'
import { RuleSet, isRandomFile, type Rules } from '../rules/index.js'
import { SegmentScanner, scanOverviewText, type EdgeSegment } from './scan.js'
import { encodeChunk, type ChunkFile } from '../chunks/index.js'

/** A release's files from one batch (REBUILD.md, chunk store): what the catalog adds up, and the encoded chunk. */
export interface ReleaseChunk {
  key: string
  name: string
  source: string
  fileTotal: number | null
  poster: string
  groups: string[]
  postedAt: number | null
  files: number
  segments: number
  bytes: number
  firstArticle: number
  lastArticle: number
  /** A file with a readable name; a par2 file. */
  readable: boolean
  hasPar2: boolean
  payload: Buffer
}

export interface DroppedChunk extends ReleaseChunk {
  rule: string
  ruleKey: string
}

export type CandidateKind = 'archive' | 'par2' | 'lone' | '7z-tail'

/** An article phase 2 should read (REBUILD.md, probe queue). */
export interface ProbeCandidate {
  releaseKey: string
  kind: CandidateKind
  messageId: string
  file: string
}

/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
export interface RuleCount {
  rule: 'crosspost' | 'obfuscated' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
  key: string
  articles: number
  files: number
}

/**
 * A file cut off by the range's edges, for the neighbouring range: its
 * details once, and the segments this range had, as
 * [part, message-id, bytes, article number].
 */
export interface Fragment {
  fileKey: string
  name: string
  poster: string
  subject: string
  totalParts: number
  groups: string[]
  release: { key: string; name: string; source: string; fileTotal?: number } | null
  postedAt: number | null
  segments: [number, string, number, number][]
}

export interface EdgeBatch {
  /** The range's resume point this batch starts at. */
  from: number
  /** First article number not covered yet. */
  next: number
  headers: number
  releases: ReleaseChunk[]
  samples: DroppedChunk[]
  candidates: ProbeCandidate[]
  counts: RuleCount[]
  fragments: Fragment[]
}

export interface RangeIndexerOptions {
  group: string
  /** Inclusive article numbers. */
  start: number
  end: number
  rules: Rules
  /** Chunks without a new part before an open file is given up (as a fragment near an edge). */
  idleChunks?: number
  /** Within this many articles of an edge, a cut-off file is kept as fragments. */
  edgeArticles?: number
}

const MAX_CANDIDATES = 4

/**
 * One leased range on an edge worker (REBUILD.md, part 2): overview lines
 * in, chunk by chunk and in order; releases, probe candidates, rule counts
 * and edge fragments out, per batch (`take`). Holds only the files still
 * waiting for parts.
 */
export class RangeIndexer {
  private aggregator = new FileAggregator<EdgeSegment>()
  private scanner = new SegmentScanner()
  private segments: EdgeSegment[] = []
  private rules: RuleSet
  private batch: EdgeBatch
  private counts = new Map<string, RuleCount>()
  private readonly idle: number
  private readonly edge: number

  constructor(private options: RangeIndexerOptions) {
    this.rules = new RuleSet(options.rules)
    this.idle = options.idleChunks ?? 5
    this.edge = options.edgeArticles ?? 50_000
    this.batch = this.empty(options.start)
  }

  /** One chunk of overview lines (parsed); `next` is the first article after it. */
  add(overviews: Overview[], next: number) {
    for (const [n, h] of overviews) this.line(n, h.subject ?? '', h.from ?? '', h.date ?? '', h['message-id'] ?? '', h[':bytes'] ?? '', h.xref ?? '')
    this.endChunk(next)
  }

  /** One chunk as the XOVER reply's text (the worker's path: no object per line). */
  addText(text: string, next: number) {
    scanOverviewText(text, this.handleLine)
    this.endChunk(next)
  }

  private handleLine = (n: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) =>
    this.line(n, subject, from, date, messageId, bytes, xref)

  private line(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) {
    if (!Number.isFinite(articleNumber) || !messageId) return
    const { group, rules } = this.options
    this.batch.headers++
    const owner = xref ? this.scanner.owner(xref, group, rules.priorities) : group
    if (owner !== group) {
      this.count('crosspost', owner, 1, 0)
      return
    }
    const segment = this.scanner.segment(articleNumber, subject, from, date, messageId, bytes, xref, group)
    if (segment) this.segments.push(segment)
    else this.count('obfuscated', '', 1, 0)
  }

  private endChunk(next: number) {
    this.files(this.aggregator.add(this.segments))
    this.segments = []
    this.cutOff(this.aggregator.evict(this.idle))
    this.batch.next = next
  }

  /** The end of the range: every open file is given up. */
  finish() {
    this.cutOff(this.aggregator.evict(0))
    this.batch.next = this.options.end + 1
  }

  /** What was found since the last take, and a fresh batch. */
  take(): EdgeBatch {
    this.judgeReleases()
    this.scanner.clear()
    const out = this.batch
    out.counts = [...this.counts.values()]
    this.counts.clear()
    this.batch = this.empty(out.next)
    return out
  }

  get openFiles() {
    return this.aggregator.openFiles
  }

  /** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
  private completed = new Map<string, AggregatedFile<EdgeSegment>[]>()

  private files(files: AggregatedFile<EdgeSegment>[]) {
    for (const file of files) {
      const key = releaseKeyOf(file.segments[0])
      const list = this.completed.get(key)
      if (list) list.push(file)
      else this.completed.set(key, [file])
    }
  }

  /** Judges and encodes the releases completed in this batch. */
  private judgeReleases() {
    const byRelease = this.completed
    this.completed = new Map()
    for (const [key, list] of byRelease) {
      const verdict = this.rules.judge(
        key,
        list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId }))
      )
      if (verdict.rule === null) {
        this.batch.releases.push(chunkOf(key, list))
        this.batch.candidates.push(...candidatesOf(key, list))
        continue
      }
      this.count(verdict.rule, verdict.key, list.reduce((n, f) => n + f.segments.length, 0), list.length)
      if (verdict.sample) this.batch.samples.push({ ...chunkOf(key, list), rule: verdict.rule, ruleKey: verdict.key })
    }
  }

  /** Given-up segments: kept per file near the range's edges (the neighbour may complete them), counted otherwise. */
  private cutOff(segments: EdgeSegment[]) {
    const { start, end } = this.options
    const byFile = new Map<string, Fragment>()
    for (const s of segments) {
      const n = Number(s.articleNumber)
      if (!Number.isFinite(n) || (n >= start + this.edge && n <= end - this.edge)) {
        this.count('incomplete', '', 1, 0)
        continue
      }
      let fragment = byFile.get(s.fileKey)
      if (!fragment) {
        fragment = fragmentOf(s)
        byFile.set(s.fileKey, fragment)
        this.batch.fragments.push(fragment)
      }
      fragment.segments.push([s.number, s.messageId, s.bytes, n])
      const t = s.postedAt
      if (Number.isFinite(t) && (fragment.postedAt === null || t < fragment.postedAt)) fragment.postedAt = t
    }
  }

  private count(rule: RuleCount['rule'], key: string, articles: number, files: number) {
    const id = `${rule}\n${key}`
    const c = this.counts.get(id)
    if (c) {
      c.articles += articles
      c.files += files
    } else {
      this.counts.set(id, { rule, key, articles, files })
    }
  }

  private empty(from: number): EdgeBatch {
    return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
  }
}

/** The release key; a file without a recognisable release becomes a release of its own. */
function releaseKeyOf(s: EdgeSegment): string {
  return s.file.release?.key ?? createHash('sha1').update(`file\n${s.fileKey}`).digest('hex')
}

function postedOf(segments: EdgeSegment[]): number | null {
  let min: number | null = null
  for (const s of segments) {
    const t = s.postedAt
    if (Number.isFinite(t) && (min === null || t < min)) min = t
  }
  return min
}

function chunkOf(key: string, files: AggregatedFile<EdgeSegment>[]): ReleaseChunk {
  const first = files[0].segments[0].file
  const chunkFiles: ChunkFile[] = files.map((f) => {
    const m = f.segments[0].file
    return { name: m.name, poster: m.poster, subject: m.subject, postedAt: postedOf(f.segments), groups: m.groups, segments: f.segments.map((x) => [x.number, x.messageId, x.bytes]) }
  })
  let segments = 0
  let bytes = 0
  let firstArticle = Number.POSITIVE_INFINITY
  let lastArticle = 0
  let postedAt: number | null = null
  for (const f of files) {
    for (const s of f.segments) {
      segments++
      bytes += s.bytes
      const n = s.articleNumber
      if (n < firstArticle) firstArticle = n
      if (n > lastArticle) lastArticle = n
    }
    const t = postedOf(f.segments)
    if (t !== null && (postedAt === null || t < postedAt)) postedAt = t
  }
  return {
    key,
    name: first.release?.name ?? first.name,
    source: first.release?.source ?? 'file',
    fileTotal: files.find((f) => f.segments[0].file.release?.fileTotal)?.segments[0].file.release?.fileTotal ?? null,
    poster: first.poster,
    groups: first.groups,
    postedAt,
    files: files.length,
    segments,
    bytes,
    firstArticle,
    lastArticle,
    readable: chunkFiles.some((f) => !isRandomFile(f.name)),
    hasPar2: chunkFiles.some((f) => /\.par2$/i.test(f.name)),
    payload: encodeChunk(chunkFiles),
  }
}

/**
 * The articles that say what a release is: the first of its first archive
 * volume, of its par2 index, of a lone random-named file, and a 7z set's
 * last volume (its listing is in the tail). At most a few per release.
 */
function candidatesOf(key: string, files: AggregatedFile<EdgeSegment>[]): ProbeCandidate[] {
  const out: ProbeCandidate[] = []
  let tail: { volume: number; file: AggregatedFile<EdgeSegment> } | null = null
  for (const f of files) {
    const name = f.segments[0].file.name
    const volume = sevenZipVolume(name)
    if (volume !== null && volume > 1) {
      if (!tail || volume > tail.volume) tail = { volume, file: f }
      continue
    }
    if (isArchiveStart(name)) out.push({ releaseKey: key, kind: 'archive', messageId: f.segments[0].messageId, file: name })
    else if (/\.par2$/i.test(name) && !isLaterVolume(name)) out.push({ releaseKey: key, kind: 'par2', messageId: f.segments[0].messageId, file: name })
  }
  if (tail) out.push({ releaseKey: key, kind: '7z-tail', messageId: tail.file.segments.at(-1)!.messageId, file: tail.file.segments[0].file.name })
  if (out.length === 0 && files.length === 1 && isRandomFile(files[0].segments[0].file.name)) {
    out.push({ releaseKey: key, kind: 'lone', messageId: files[0].segments[0].messageId, file: files[0].segments[0].file.name })
  }
  return out.slice(0, MAX_CANDIDATES)
}

function fragmentOf(s: EdgeSegment): Fragment {
  const m = s.file
  return {
    fileKey: s.fileKey,
    name: m.name,
    poster: m.poster,
    subject: m.subject,
    totalParts: m.totalParts,
    groups: m.groups,
    release: m.release ? { key: m.release.key, name: m.release.name, source: m.release.source, ...(m.release.fileTotal ? { fileTotal: m.release.fileTotal } : {}) } : null,
    postedAt: null,
    segments: [],
  }
}