usenet-indexer/lib/edge/scan.ts

import { fileKey } from '../headers/index.js'
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'

/** A file's details, shared by all of its segments (one object per file, not per article). */
export interface FileMeta {
  fileKey: string
  name: string
  poster: string
  /** The subject without its segment counter. */
  subject: string
  totalParts: number
  groups: string[]
  release: ReleaseIdentity | null
}

/** One article of a file, as lean as the edge worker can keep it. */
export interface EdgeSegment {
  fileKey: string
  number: number
  totalParts: number
  messageId: string
  bytes: number
  articleNumber: number
  /** ms since 1970, NaN when the Date header doesn't parse. */
  postedAt: number
  file: FileMeta
}

interface ParsedSubjectBase {
  name: string
  totalSegments: number
  base: string
}

const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g

/**
 * Builds segments from overview fields (REBUILD.md, edge worker), the same
 * as lib/headers' segmentFromHeader but allocating as little as it can: a
 * subject is parsed once per file (cached without its segment counter), a
 * file's details are one shared object, and the Xref is read without
 * building arrays. Garbage collection was ~20% of the worker's time
 * (LOG.md 2026-10-09). The caches live until `clear`, once per batch.
 */
export class SegmentScanner {
  private subjects = new Map<string, ParsedSubjectBase | null>()
  private files = new Map<string, FileMeta>()

  clear() {
    this.subjects.clear()
    this.files.clear()
  }

  /** The owner of an article by its Xref field: the indexed group with the lowest priority, else `group`. */
  owner(xref: string, group: string, priorities: Record<string, number>): string {
    // "news.example a.b.test:123 a.b.x:456": the first token is the server.
    let at = xref.indexOf(' ')
    if (at === -1) return group
    let owner: string | null = null
    let best = Number.POSITIVE_INFINITY
    while (at !== -1) {
      while (xref.charCodeAt(at) === 32) at++
      if (at >= xref.length) break
      const end = xref.indexOf(' ', at)
      const colon = xref.lastIndexOf(':', end === -1 ? xref.length : end)
      if (colon > at) {
        const name = xref.slice(at, colon)
        const p = priorities[name]
        if (p !== undefined && p < best) {
          best = p
          owner = name
        }
      }
      at = end
    }
    return owner ?? group
  }

  /** A segment from an article's fields, or null when its subject is obfuscated. */
  segment(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string, group: string): EdgeSegment | null {
    if (!subject || !messageId) return null
    const nameEnd = subject.lastIndexOf('"')
    if (nameEnd <= 0) return null
    // The segment counter: the last "(k/N)" after the filename (a file size may follow it).
    SEGMENT_COUNTER.lastIndex = nameEnd
    let counter: RegExpExecArray | null = null
    for (let m = SEGMENT_COUNTER.exec(subject); m; m = SEGMENT_COUNTER.exec(subject)) counter = m
    if (!counter) return null
    const key = subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter[0].length)
    let parsed = this.subjects.get(key)
    if (parsed === undefined) {
      parsed = parseBase(subject, nameEnd, counter)
      this.subjects.set(key, parsed)
    }
    if (!parsed) return null
    const number = Number.parseInt(counter[1], 10)
    if (number < 1 || number > parsed.totalSegments) return null
    const id = `${from}\n${parsed.base}\n${parsed.totalSegments}`
    let file = this.files.get(id)
    if (!file) {
      file = {
        fileKey: fileKey(from, parsed.base, parsed.totalSegments),
        name: parsed.name,
        poster: from,
        subject: parsed.base,
        totalParts: parsed.totalSegments,
        groups: xrefGroups(xref, group),
        release: releaseIdentity({ poster: from, subject: parsed.base, name: parsed.name, weak: false }),
      }
      this.files.set(id, file)
    }
    return {
      fileKey: file.fileKey,
      number,
      totalParts: file.totalParts,
      messageId: messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId,
      bytes: Number.parseInt(bytes, 10) || 0,
      articleNumber,
      postedAt: Date.parse(date),
      file,
    }
  }
}

/** As parseSubject: the name, the segment count and the base, from a subject whose counter was found. */
function parseBase(subject: string, nameEnd: number, counter: RegExpExecArray): ParsedSubjectBase | null {
  const nameStart = subject.lastIndexOf('"', nameEnd - 1)
  if (nameStart === -1) return null
  const name = subject.slice(nameStart + 1, nameEnd)
  if (!name) return null
  const totalSegments = Number.parseInt(counter[2], 10)
  if (totalSegments < 1) return null
  const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)).replace(/\s+/g, ' ').trim()
  return { name, totalSegments, base }
}

function xrefGroups(xref: string, group: string): string[] {
  const groups: string[] = []
  const tokens = xref.trim().split(/\s+/)
  for (let i = 1; i < tokens.length; i++) {
    const colon = tokens[i].lastIndexOf(':')
    if (colon > 0) groups.push(tokens[i].slice(0, colon))
  }
  return groups.length > 0 ? groups : [group]
}

/**
 * Calls `line` for each overview line of an XOVER reply's text, with its
 * fields as strings (no object per line): number, subject, from, date,
 * message-id, :bytes, and the Xref without its "Xref: " name.
 */
export function scanOverviewText(
  text: string,
  line: (number: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) => void
) {
  let start = 0
  const length = text.length
  while (start < length) {
    let end = text.indexOf('\n', start)
    if (end === -1) end = length
    const stop = end > start && text.charCodeAt(end - 1) === 13 ? end - 1 : end
    if (stop > start) {
      // number, subject, from, date, message-id, references, bytes, lines, xref
      const t1 = text.indexOf('\t', start)
      const t2 = t1 === -1 ? -1 : text.indexOf('\t', t1 + 1)
      const t3 = t2 === -1 ? -1 : text.indexOf('\t', t2 + 1)
      const t4 = t3 === -1 ? -1 : text.indexOf('\t', t3 + 1)
      const t5 = t4 === -1 ? -1 : text.indexOf('\t', t4 + 1)
      const t6 = t5 === -1 ? -1 : text.indexOf('\t', t5 + 1)
      const t7 = t6 === -1 ? -1 : text.indexOf('\t', t6 + 1)
      const t8 = t7 === -1 ? -1 : text.indexOf('\t', t7 + 1)
      if (t5 !== -1 && t5 < stop) {
        const field = (a: number, b: number) => (a === -1 || a >= stop ? '' : text.slice(a + 1, b === -1 || b > stop ? stop : b))
        let xref = t8 === -1 || t8 >= stop ? '' : field(t8, text.indexOf('\t', t8 + 1))
        if (xref.startsWith('Xref:')) xref = xref.slice(xref.charCodeAt(5) === 32 ? 6 : 5)
        line(Number.parseInt(text.slice(start, t1), 10), text.slice(t1 + 1, t2), field(t2, t3), field(t3, t4), field(t4, t5), field(t6, t7), xref)
      }
    }
    start = end + 1
  }
}