usenet-indexer/lib/edge/scan.ts

import { fileKey, nameBesideQuotes, unquotedName } from '../headers/index.js'
import { releaseIdentity, type ReleaseIdentity } from '../release/index.js'

/** A file's details, shared by all of its segments (one object per file, not per article). */
export interface FileMeta {
  fileKey: string
  name: string
  poster: string
  /** The subject without its segment counter. */
  subject: string
  totalParts: number
  groups: string[]
  release: ReleaseIdentity | null
}

/** One article of a file, as lean as the edge worker can keep it. */
export interface EdgeSegment {
  fileKey: string
  number: number
  totalParts: number
  messageId: string
  bytes: number
  articleNumber: number
  /** ms since 1970, NaN when the Date header doesn't parse. */
  postedAt: number
  file: FileMeta
}

/**
 * A copy of a string that doesn't keep the string it was cut from alive: a
 * slice of a chunk's text otherwise holds on to the whole text (tens of MB)
 * for as long as the file it belongs to stays open. External memory was
 * ~570 MB of a worker's 1.1 GB (LOG.md 2026-10-09).
 */
function own(text: string): string {
  return (' ' + text).slice(1)
}

interface Counter {
  index: number
  length: number
  segment: number
  total: number
}

/** The last "(k/N)" after `after` (the filename's closing quote), scanning back from the end; null when there is none. */
function findCounter(subject: string, after: number): Counter | null {
  let open = subject.length
  while (true) {
    open = subject.lastIndexOf('(', open - 1)
    if (open <= after) return null
    let i = open + 1
    let segment = 0
    let digits = 0
    for (let c = subject.charCodeAt(i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) {
      segment = segment * 10 + (c - 48)
      digits++
    }
    if (digits === 0 || subject.charCodeAt(i) !== 47) continue
    let total = 0
    digits = 0
    for (let c = subject.charCodeAt(++i); c >= 48 && c <= 57; c = subject.charCodeAt(++i)) {
      total = total * 10 + (c - 48)
      digits++
    }
    if (digits === 0 || subject.charCodeAt(i) !== 41) continue
    return { index: open, length: i + 1 - open, segment, total }
  }
}

/**
 * Builds segments from overview fields (REBUILD.md, edge worker), the same
 * as lib/headers' segmentFromHeader but allocating as little as it can: a
 * subject is parsed once per file (cached without its segment counter), a
 * file's details are one shared object, and the Xref is read without
 * building arrays. Garbage collection was ~20% of the worker's time
 * (LOG.md 2026-10-09). The caches live until `clear`, once per batch.
 */
export class SegmentScanner {
  /** By poster and subject without its counter: the file's details, or null for a subject that names no file. */
  private files = new Map<string, FileMeta | null>()

  clear() {
    this.files.clear()
  }

  /** The owner of an article by its Xref field: the indexed group with the lowest priority, else `group`. */
  owner(xref: string, group: string, priorities: Record<string, number>): string {
    // "news.example a.b.test:123 a.b.x:456": the first token is the server.
    let at = xref.indexOf(' ')
    if (at === -1) return group
    let owner: string | null = null
    let best = Number.POSITIVE_INFINITY
    while (at !== -1) {
      while (xref.charCodeAt(at) === 32) at++
      if (at >= xref.length) break
      const end = xref.indexOf(' ', at)
      const colon = xref.lastIndexOf(':', end === -1 ? xref.length : end)
      if (colon > at) {
        const name = xref.slice(at, colon)
        const p = priorities[name]
        if (p !== undefined && p < best) {
          best = p
          owner = name
        }
      }
      at = end
    }
    return owner ?? group
  }

  /** A segment from an article's fields, or null when its subject is obfuscated. */
  segment(articleNumber: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string, group: string): EdgeSegment | null {
    if (!subject || !messageId) return null
    // After the last quote; without one, anywhere (unquotedName finds the file name).
    const nameEnd = subject.lastIndexOf('"')
    const counter = findCounter(subject, nameEnd)
    if (!counter) return null
    const key = from + '\u0001' + subject.slice(0, counter.index) + '\u0000' + subject.slice(counter.index + counter.length)
    let file = this.files.get(key)
    if (file === undefined) {
      file = this.describe(subject, nameEnd, counter, from, xref, group)
      this.files.set(own(key), file)
    }
    if (!file || counter.segment < 1 || counter.segment > file.totalParts) return null
    return {
      fileKey: file.fileKey,
      number: counter.segment,
      totalParts: file.totalParts,
      messageId: own(messageId.charCodeAt(0) === 60 ? messageId.slice(1, -1) : messageId),
      bytes: Number.parseInt(bytes, 10) || 0,
      articleNumber,
      postedAt: Date.parse(date),
      file,
    }
  }

  /** A file's details from its first article seen (as parseSubject and segmentFromHeader). */
  private describe(subject: string, nameEnd: number, counter: Counter, from: string, xref: string, group: string): FileMeta | null {
    const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1
    const quoted = nameStart !== -1 ? subject.slice(nameStart + 1, nameEnd) : null
    const found = quoted !== null ? (nameBesideQuotes(quoted, subject.slice(0, counter.index)) ?? quoted) : unquotedName(subject.slice(0, counter.index))
    if (!found || counter.total < 1) return null
    const name = own(found)
    const base = own((subject.slice(0, counter.index) + subject.slice(counter.index + counter.length)).replace(/\s+/g, ' ').trim())
    const poster = own(from)
    return {
      fileKey: fileKey(poster, base, counter.total),
      name,
      poster,
      subject: base,
      totalParts: counter.total,
      groups: xrefGroups(xref, group),
      release: releaseIdentity({ poster, subject: base, name, weak: false }),
    }
  }
}

function xrefGroups(xref: string, group: string): string[] {
  const groups: string[] = []
  const tokens = xref.trim().split(/\s+/)
  for (let i = 1; i < tokens.length; i++) {
    const colon = tokens[i].lastIndexOf(':')
    if (colon > 0) groups.push(own(tokens[i].slice(0, colon)))
  }
  return groups.length > 0 ? groups : [group]
}

/**
 * Calls `line` for each overview line of an XOVER reply's text, with its
 * fields as strings (no object per line): number, subject, from, date,
 * message-id, :bytes, and the Xref without its "Xref: " name.
 */
export function scanOverviewText(
  text: string,
  line: (number: number, subject: string, from: string, date: string, messageId: string, bytes: string, xref: string) => void
) {
  let start = 0
  const length = text.length
  while (start < length) {
    let end = text.indexOf('\n', start)
    if (end === -1) end = length
    const stop = end > start && text.charCodeAt(end - 1) === 13 ? end - 1 : end
    if (stop > start) {
      // number, subject, from, date, message-id, references, bytes, lines, xref
      const t1 = text.indexOf('\t', start)
      const t2 = t1 === -1 ? -1 : text.indexOf('\t', t1 + 1)
      const t3 = t2 === -1 ? -1 : text.indexOf('\t', t2 + 1)
      const t4 = t3 === -1 ? -1 : text.indexOf('\t', t3 + 1)
      const t5 = t4 === -1 ? -1 : text.indexOf('\t', t4 + 1)
      const t6 = t5 === -1 ? -1 : text.indexOf('\t', t5 + 1)
      const t7 = t6 === -1 ? -1 : text.indexOf('\t', t6 + 1)
      const t8 = t7 === -1 ? -1 : text.indexOf('\t', t7 + 1)
      if (t5 !== -1 && t5 < stop) {
        const field = (a: number, b: number) => (a === -1 || a >= stop ? '' : text.slice(a + 1, b === -1 || b > stop ? stop : b))
        let xref = t8 === -1 || t8 >= stop ? '' : field(t8, text.indexOf('\t', t8 + 1))
        if (xref.startsWith('Xref:')) xref = xref.slice(xref.charCodeAt(5) === 32 ? 6 : 5)
        line(Number.parseInt(text.slice(start, t1), 10), text.slice(t1 + 1, t2), field(t2, t3), field(t3, t4), field(t4, t5), field(t6, t7), xref)
      }
    }
    start = end + 1
  }
}