usenet-indexer/lib/rules/rule_set.ts

import { explored, signatureKey, uploadTool } from './upload_tools.js'
import { fileStem, looksRandom } from '../release/index.js'

/**
 * What a worker drops, and why (REBUILD.md, rules and recovery). Sent with
 * every lease; `version` is recorded on the range, so a changed rule set
 * shows which ranges to read again.
 */
export interface Rules {
  version: number
  /** Indexed groups and their priority: a crosspost belongs to the indexed group in its Xref with the lowest number. */
  priorities: Record<string, number>
  /** Dropped signatures, as "tool\tgroup" (GarbageFilter's table). */
  droppedSignatures: string[]
  /**
   * Dropped classes of unnamed releases, as "tool\tgroup\tklass" (lib/release/classes.ts; the pending ladder's record,
   * LOG.md (130)): the edge keeps them out of the pending store, the explored sample excepted.
   */
  droppedClasses: string[]
  /** Tools whose random-named uploads are dropped in every group (named < 1%; LOG.md 2026-10-08 (5)). */
  droppedTools: string[]
  /** A random-named file of at most this many parts: a random subject per article, needs bodies. */
  perArticleMaxParts: number
  /** Share of each rule's drops kept in full as a sample (by a hash of the release key). */
  samplePercent: number
  /**
   * Drop releases of bare executables (malware floods posted under film and
   * software names; 92% of the catalog rows of the first test ranges, LOG.md
   * 2026-10-09). Genuine software comes in archives.
   */
  dropExecutables: boolean
  /**
   * Posters whose uploads are dropped whatever they are: fragments of the poster lowercased without punctuation
   * ("@__Celeb_Movie__@" → "celebmovie"). The CelebMovie family's scraped celebrity pictures, the same files again and
   * again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)).
   */
  droppedPosters: string[]
  /**
   * Drop settled releases of one unsplit archive without a par2, posted to 2+ groups, under STUB_MAX_BYTES, whose
   * articles are at most STUB_ARTICLE_BYTES with the release at least STUB_MIN_BYTES, or posted to 4+ groups, or by
   * a webmail poster: the exe-stub and recycled-installer floods under film and software names (55% of the catalog
   * on 2026-10-30; 99.3% of the known fakes, 0 known content; LOG.md (123)-(126), (128)).
   */
  dropStubArchives: boolean
  /**
   * Drop one-article archives of NONSENSE_MIN_BYTES to NONSENSE_MAX_BYTES under a release-like, unscrambled name:
   * the newznzb.* pieces (9% of the catalog; LOG.md (123), (125)).
   */
  dropNonsensePieces: boolean
}

export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
  version: 1,
  droppedTools: ['ngPost', 'hex32', 'yuun', 'JBinUp', 'random'],
  perArticleMaxParts: 3,
  samplePercent: 1,
  dropExecutables: true,
  droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
  dropStubArchives: true,
  dropNonsensePieces: true,
  droppedClasses: [],
}

/** The stub-archive rule's bounds (edge/src/names.rs): Power-Post's 250 kB default article, the fakes' size span. */
export const STUB_ARTICLE_BYTES = 262144
export const STUB_MIN_BYTES = 1_000_000
export const STUB_MAX_BYTES = 100_000_000
export const STUB_WIDE_GROUPS = 4
/** The nonsense-piece rule's bounds: under 100 kB are cue/log sidecars and adverts, 1 MB and up is rule 1's. */
export const NONSENSE_MIN_BYTES = 100_000
export const NONSENSE_MAX_BYTES = 1_000_000

export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article' | 'stub-archive' | 'nonsense-piece'; key: string; sample: boolean }

/** A file as the judge sees it: its name, parts, groups, poster, and the articles seen with their bytes. */
export interface JudgedFile {
  name: string
  totalParts: number
  groups: string[]
  firstMessageId: string
  poster: string
  segments: number
  bytes: number
}

/** A poster as droppedPosters matches it: lowercased, letters and digits only. */
export function posterWords(poster: string): string {
  return poster.replace(/[^A-Za-z0-9]/g, '').toLowerCase()
}

/** Files that come along with a release's content: they don't make it one of executables or not. */
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
const EXECUTABLE = /\.(exe|scr|com|pif|bat|cmd|vbs|lnk)\s*$/i
/** One zip, rar or 7z that is not a volume of a set. */
const ARCHIVE = /\.(zip|rar|7z)$/i
/** "cneg" is "part" in ROT13: the a.b.erotica posts named that way are volumes of passworded videos (LOG.md (128)). */
const VOLUME = /\.((part|cneg)\d+\.rar|r\d\d|z\d\d|7z\.\d\d\d|\d\d\d)$/i
/** A poster at a free mail provider: the stub floods' throwaway accounts. */
const WEBMAIL = /@(gmail|googlemail|yahoo|hotmail|outlook|live|msn|aol|gmx|web|t-online|freenet|mail|yandex|icloud|me|hushmail|protonmail|libero|orange|free|arcor|wanadoo|ymail|rocketmail|inbox|mail\.ru|bk|list)\./i
/** A name that reads as a release: a year, a scene tag, a group suffix or dotted words. */
const RELEASE_LIKE = /(\b(19|20)\d\d\b|[._-](720p|1080p|2160p|x264|x265|hevc|bluray|web-?dl|webrip|hdtv|xvid|s\d\de\d\d|german|multi|crack|repack|proper|flac|mp3|xxx|mp4|retail|keygen|portable|incl|v\d+(\.\d+)+)\b|-[A-Za-z0-9]{2,}$|\.[A-Za-z0-9]+\.[A-Za-z0-9]+\.)/i

/**
 * Groups whose communities post real single small archives (karaoke zips, e-books, comics, tabs, fonts) the way the
 * stub floods do (Power-Post defaults, a crosspost or two): a release touching one is exempt from the stub-archive
 * rule. Costs 16 of 32,308 known fakes in the 217k sample (LOG.md (127)).
 */
const STUB_EXEMPT_GROUPS = /(sound|mp3|karaoke|music|audio|flac|lossless|e-?book|epub|comic|guitar|font|magazine)/i

/** The single unsplit archive a release consists of, or null. */
function singleArchive(files: JudgedFile[]): JudgedFile | null {
  return files.length === 1 && ARCHIVE.test(files[0].name) && !VOLUME.test(files[0].name) ? files[0] : null
}

/**
 * The legs of the stub-archive rule a settled single archive under STUB_MAX_BYTES (the fakes stop at ~90 MB;
 * hash-named uploads, real albums and passworded videos above it don't) trips: "A" (articles <= STUB_ARTICLE_BYTES,
 * release >= STUB_MIN_BYTES: a one-article release always has small articles, so e-books and sidecars are exempt),
 * "X" (4+ groups) and "W" (webmail poster). Empty when none.
 */
export function stubArchiveLegs(f: JudgedFile): string {
  if (f.groups.length < 2 || f.bytes >= STUB_MAX_BYTES || f.groups.some((g) => STUB_EXEMPT_GROUPS.test(g))) return ''
  let legs = ''
  if (f.bytes / Math.max(f.segments, 1) <= STUB_ARTICLE_BYTES && f.bytes >= STUB_MIN_BYTES) legs += 'A'
  if (f.groups.length >= STUB_WIDE_GROUPS) legs += 'X'
  if (WEBMAIL.test(f.poster)) legs += 'W'
  return legs
}

/** An e-book by its name ("Author - Title (2020) (epub).rar"): one small article is a book, not a nonsense piece. */
const EBOOK = /\b(epub|mobi|azw3?|pdf|djvu|cbr|cbz|e-?books?)\b/i

/**
 * A nonsense piece: one article, one small archive under a release-like, unscrambled name without spaces (the
 * newznzb.* pieces are scene-styled or random with a "-XX" suffix; books and albums are written with spaces).
 */
export function isNonsensePiece(f: JudgedFile): boolean {
  return (
    f.totalParts === 1 &&
    f.segments === 1 &&
    f.bytes >= NONSENSE_MIN_BYTES &&
    f.bytes < NONSENSE_MAX_BYTES &&
    !f.name.includes(' ') &&
    RELEASE_LIKE.test(fileStem(f.name)) &&
    !EBOOK.test(f.name) &&
    !isScrambledFile(f.name) &&
    !isRandomFile(f.name)
  )
}

export function isRandomFile(name: string) {
  return looksRandom(fileStem(name))
}

/**
 * At least half the letters and digits of a file's stem in tokens of 5 or more that switch case (lower to upper) or
 * between letters and digits 3 times or more ("nYDeXeF49dieZ1J7flK-JFzZ3j.ZEDsr", "bslXlwgPiqZadhRa"): the names a
 * random-subject-per-article upload gives each article that looksRandom misses for their separators.
 */
export function isScrambledFile(name: string) {
  let total = 0
  let scrambled = 0
  for (const token of fileStem(name).split(/[^A-Za-z0-9]+/)) {
    total += token.length
    let switches = 0
    for (let i = 1; i < token.length; i++) {
      const [a, b] = [token[i - 1], token[i]]
      if ((/[a-z]/.test(a) && /[A-Z]/.test(b)) || /\d/.test(a) !== /\d/.test(b)) switches++
    }
    if (token.length >= 5 && switches >= 3) scrambled += token.length
  }
  return total >= 8 && scrambled * 2 >= total
}

/** The owner of an article posted to `groups` (its Xref), or null when none of them is indexed. */
export function ownerOf(groups: string[], priorities: Record<string, number>): string | null {
  let owner: string | null = null
  for (const g of groups) {
    const p = priorities[g]
    if (p !== undefined && (owner === null || p < priorities[owner])) owner = g
  }
  return owner
}

export class RuleSet {
  private signatures: Set<string>
  readonly classes: Set<string>
  private tools: Set<string>
  private posters: string[]
  constructor(readonly rules: Rules) {
    this.posters = rules.droppedPosters ?? []
    this.signatures = new Set(rules.droppedSignatures)
    this.classes = new Set(rules.droppedClasses ?? [])
    this.tools = new Set(rules.droppedTools)
  }

  /**
   * A release's verdict from its files: dropped when its poster is; when its content is bare
   * executables; when it is a settled single archive that is a stub or a nonsense piece; else kept as a whole when
   * any file has a readable name; else dropped as garbage when its signature or tool is dropped, as per-article
   * random subjects when every file has few parts. `settled`: the window closed on the release (a par2 can still
   * join an open one), which the single-archive rules need.
   */
  judge(releaseKey: string, files: JudgedFile[], settled = true): Verdict {
    const sample = explored(releaseKey, this.rules.samplePercent)
    if (this.posters.length > 0 && files.length > 0) {
      const words = posterWords(files[0].poster)
      const dropped = this.posters.find((p) => words.includes(p))
      if (dropped) return { rule: 'poster', key: dropped, sample }
    }
    if (this.rules.dropExecutables) {
      const content = files.filter((f) => !SIDECAR.test(f.name))
      if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
        return { rule: 'executable', key: EXECUTABLE.exec(content[0].name)![1].toLowerCase(), sample }
      }
    }
    const archive = settled ? singleArchive(files) : null
    if (archive) {
      if (this.rules.dropStubArchives) {
        const legs = stubArchiveLegs(archive)
        if (legs !== '') return { rule: 'stub-archive', key: legs, sample }
      }
      if (this.rules.dropNonsensePieces && isNonsensePiece(archive)) return { rule: 'nonsense-piece', key: uploadTool(archive.firstMessageId), sample }
    }
    if (files.some((f) => !isRandomFile(f.name))) return { rule: null }
    const first = files[0]
    const tool = uploadTool(first.firstMessageId)
    // Garbage first: a dropped tool or signature isn't worth reading bodies for either.
    const group = first.groups[0] ?? ''
    if (this.tools.has(tool) || this.signatures.has(signatureKey(tool, group))) return { rule: 'garbage', key: `${tool} ${group}`, sample }
    if (files.every((f) => f.totalParts <= this.rules.perArticleMaxParts)) return { rule: 'per-article', key: tool, sample }
    return { rule: null }
  }
}