usenet-indexer/lib/release/identity.ts

import { hash as digest } from 'node:crypto'
import { looksLikeFilename } from '../headers/file_key.js'
import { looksRandom } from './random.js'

export interface ReleaseIdentity {
  key: string
  /** Human-readable name: the subject text or the shared filename stem. */
  name: string
  /** Number of files announced by the `[i/M]` counter, when present. */
  fileTotal?: number
  /** Which rule recognised the release. */
  source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot'
}

export interface ReleaseInput {
  poster: string
  /** Subject without segment counter (the binary's `subject`). */
  subject: string
  /** Filename of the binary. */
  name: string
  /** True when the identity came from the yEnc body (obfuscated post). */
  weak: boolean
}

// Suffixes that differ between the files of one release. Stripped repeatedly,
// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x".
const SUFFIXES = [
  /\.vol\d+[+-]\d+\.par2$/i,
  /\.par2$/i,
  /\.part\d+\.rar$/i,
  /\.r\d{2,3}$/i,
  /\.rar$/i,
  /\.7z(\.\d{3})?$/i,
  /\.zip(\.\d{3})?$/i,
  // Split volumes (x.mkv.001), not codec names (H.264, H.265).
  /(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i,
  /\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
  /\.tar$/i,
  /\.(zst|gz|xz|bz2)$/i,
  /\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
]

/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
export function fileStem(name: string): string {
  let stem = name.trim()
  let changed = true
  while (changed) {
    changed = false
    for (const suffix of SUFFIXES) {
      const next = stem.replace(suffix, '')
      if (next !== stem && next.length > 0) {
        stem = next
        changed = true
      }
    }
  }
  return stem
}

/**
 * What remains of a subject once the filename, counters, "yEnc" and sizes are
 * removed. Posters that name the release outside the quotes put it here.
 */
/** A file counter's text when its first number is at most its second, else the text itself (a date like [1948-12] stays). */
const counter = (text: string, i: string, n: string) => (Number(i) <= Number(n) ? ' ' : text)

export function subjectRemainder(subject: string): string {
  return subject
    .replace(/"[^"]*"/g, ' ')
    .replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
    // Counters written out: "File 027 of 132", "[21 of 25]", "104 of 472", "[2239-6389]", "02/17",
    // "0097-1574" (each image of a set was a release of its own; LOG.md 2026-10-09 (9)).
    .replace(/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b[\])]?/gi, counter)
    .replace(/\[(\d+)-(\d+)\]/g, counter)
    .replace(/\b(\d+)\/(\d+)\b/g, counter)
    .replace(/\b(0\d*)-(\d+)\b/g, counter)
    // File sizes ("76.88 MB", "335555 Bytes") differ per file.
    .replace(/\b\d+(?:[.,]\d+)?\s*(?:[kmgt]i?b|bytes?)\b/gi, ' ')
    // File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
    .replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
    .replace(/\byEnc\b/gi, ' ')
    .replace(/[\s\-–.:|]+/g, ' ')
    // What's left of "["name.jpg" yEnc]" and of counters and sizes in brackets.
    .replace(/ ?(?:\[ ?\]|\( ?\)) ?/g, ' ')
    .replace(/ {2,}/g, ' ')
    .trim()
}

/**
 * Posting tools put random per-file tokens and poster IDs in subjects (e.g.
 * `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
 * name once those are removed identifies a release.
 */
function isRandomToken(token: string): boolean {
  const t = token.replace(/^\[|\]$/g, '')
  if (t.length < 6) return false
  if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true
  return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t)
}

/**
 * A stable per-post token: one short run of letters/digits with at least one
 * digit. Requiring a digit keeps constant labels like `[PRiVATE]` from
 * grouping different posts of the same poster.
 */
export function isPostToken(remainder: string): boolean {
  const compact = remainder.replace(/[\s\[\]()_-]+/g, '')
  return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact)
}

export function namesRelease(remainder: string): boolean {
  const words = remainder
    .split(/\s+/)
    .filter((w) => w && !isRandomToken(w))
    .join(' ')
  // At least two words of letters, joined the way release names are.
  return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words)
}

function hash(...parts: (string | number | undefined)[]) {
  return digest('sha1', parts.map((p) => p ?? '').join('\n'))
}

/** Image files: a set's images are named one by one ("IMG_3086.jpg"); its subject names the set. */
const IMAGE = /\.(jpe?g|png|gif|bmp|webp|tiff?)$/i

/** Audio tracks: parts of an album or audiobook that the subject names ("072 - Die Teufelsuhr.MP3 - by DeckaR"). */
const AUDIO = /\.(mp3|flac|m4a|m4b|ogg|opus|wav|aac|wma|ape)(?:$|[^a-z0-9])/i

/** Documents and comics: each file is an item of its own, whatever the subject says ("cw-misc [020/109] "Action Comics 710.cbr""). */
const DOCUMENT = /\.(pdf|epub|mobi|azw3?|djvu|cbr|cbz|cb7|chm|lit|docx?)$/i

/**
 * The total of the subject's file counter, in the forms subjectRemainder
 * removes, the first form found first: [i/M] or (i/M), "i of M", [i-M], i/M,
 * 0i-M (the last four only when i <= M).
 */
export function counterTotal(subject: string): number | undefined {
  const s = subject.replace(/"[^"]*"/g, ' ')
  const plain = /\[(\d+)\/(\d+)\]|\((\d+)\/(\d+)\)/.exec(s)
  if (plain) return Number(plain[2] ?? plain[4])
  for (const pattern of [/[[(]?\b(?:file\s+)?(\d+)\s*of\s*(\d+)\b/gi, /\[(\d+)-(\d+)\]/g, /\b(\d+)\/(\d+)\b/g, /\b(0\d*)-(\d+)\b/g]) {
    for (const m of s.matchAll(pattern)) if (Number(m[1]) <= Number(m[2])) return Number(m[2])
  }
  return undefined
}

/** A name that says what it is: 12+ letters and digits, two words of 3+ letters. */
function describes(stem: string): boolean {
  return stem.replace(/[^A-Za-z0-9]/g, '').length >= 12 && /[A-Za-z]{3,}[^A-Za-z0-9]+[A-Za-z]{3,}/.test(stem)
}

/** Words of a subject that say nothing about what is posted: hosts, posting boilerplate. */
const TAG_WORDS = new Set(['www', 'http', 'https', 'com', 'net', 'org', 'info', 'the', 'and', 'presents', 'posted', 'post', 'powered', 'yenc', 'part', 'rar', 'par2', 'nzb', 'voor', 'for', 'from'])

/**
 * Whether the subject's text outside the quotes is a poster's constant tag
 * rather than about the file: it shares no word with the file's stem. A word
 * of 4+ letters may be part of the stem ("ketchup" in "ketchup_dance"); a
 * 3-letter word or a number ("35" in "35_China_dl") must be a whole word;
 * boilerplate doesn't count. A number of 5+ digits is a per-post id
 * ("FTD 769997", "(781442)"), so never a constant tag.
 */
export function isPosterTag(remainder: string, stem: string): boolean {
  const words = stem.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
  const compact = words.replace(/ /g, '')
  const whole = new Set(words.split(' '))
  // A per-post id, also inside a word ("FTD772544").
  if (/\d{5,}/.test(remainder)) return false
  for (const word of remainder.toLowerCase().split(/[^a-z0-9]+/)) {
    if (/^\d+$/.test(word)) {
      if (word.length >= 2 && whole.has(word)) return false
      continue
    }
    if (word.length < 3 || TAG_WORDS.has(word)) continue
    if (word.length >= 4 ? compact.includes(word) : whole.has(word)) return false
  }
  return true
}

/**
 * Decides which release (download) a file belongs to. Returns null when the
 * file cannot be grouped from its name alone (random extensionless names in
 * obfuscated posts; those need par2 matching). See CONCEPT.md (D16).
 */
export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
  const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
  const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
  const stem = fileStem(input.name)

  if (input.weak) {
    // Obfuscating posters randomise the poster per article; a real filename
    // stem is specific enough on its own.
    if (!looksLikeFilename(input.name)) return null
    return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
  }

  // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
  const remainder = subjectRemainder(input.subject)
  // A document or comic is a release of its own (collections post hundreds under one subject).
  if (DOCUMENT.test(input.name)) return { key: hash('stem', input.poster, stem), name: stem.replace(DOCUMENT, '') || stem, fileTotal, source: 'stem' }
  const image = IMAGE.test(input.name)
  // The poster's constant tag ("[KiLL.dA…]---- (01/10) "Album.rar"", "(X Posted)") would make one
  // release of all their uploads: a readable file name that shares no word with it says which
  // release the file is (LOG.md 2026-10-09 (8)). Only when the stem itself describes something
  // ("Cephalgy.-.Weisses.Fleisch.2003", not a scene abbreviation like "grp-sm" under
  // "Some.Movie.2008-GRP", nor "file_id.diz"), and not for images and audio tracks, which the
  // subject names as a set or album ("IMG_3086.jpg", "072 - Die Teufelsuhr.MP3").
  const tag = !image && !AUDIO.test(input.name) && describes(stem) && !looksRandom(stem) && isPosterTag(remainder, stem)
  // An image set without a counter: the image's name up to its last number, without numbers
  // ("Kitchen Hottie 1 set233-76-lg" → "Kitchen Hottie set", "bm_jessica_blb_h087" → "bm jessica blb h").
  const prefix = image ? stem.replace(/\d+\D*$/, '').replace(/\d+/g, ' ').replace(/[\s._-]+/g, ' ').trim() : ''
  if (!tag && namesRelease(remainder)) {
    // Bounded by the counter's total, so the uploads a poster names alike are apart while the files
    // of one upload (par2 and nfo included) share it (LOG.md 2026-10-09 (9)).
    const total = counterTotal(input.subject)
    return {
      key: hash('subject', input.poster, remainder, total ?? prefix),
      name: remainder,
      fileTotal,
      source: 'subject',
    }
  }
  // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
  // [i/M] counter it groups the files of one post whose names are random.
  if (!tag && fileTotal && isPostToken(remainder)) {
    return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
  }
  // An image without words around it: its set by its name.
  if (image) return { key: hash('images', input.poster, prefix), name: prefix || stem, fileTotal, source: 'stem' }
  // The [i/M] total is informational, not part of the key: posters often
  // upload the par2 index without a counter, and it belongs to the same post.
  return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
}

/** Release of a par2 recovery set; every file listed in it belongs to it. */
export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity {
  return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' }
}

const SEPARATOR = /[._ -]/

/**
 * A release name from the files a par2 set lists. Files that differ only in
 * the middle (episodes S03E01…S03E20, parts) give their shared name, with
 * the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to
 * SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato.
 * Otherwise the longest stem, which tends to be the most descriptive (the
 * video file).
 */
export function par2ReleaseName(fileNames: string[]): string | undefined {
  const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean)
  if (stems.length === 0) return undefined
  const longest = [...stems].sort((a, b) => b.length - a.length)[0]
  if (stems.length < 2) return longest

  let prefix = stems.reduce((p, s) => {
    let i = 0
    while (i < p.length && i < s.length && p[i] === s[i]) i++
    return p.slice(0, i)
  })
  let suffix = stems.reduce((p, s) => {
    let i = 0
    while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++
    return p.slice(p.length - i)
  })
  const shortest = Math.min(...stems.map((s) => s.length))
  if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest)

  // Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token.
  const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix)
  if (season) prefix = season[1]
  else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) {
    prefix = prefix.replace(/[^._ -]*$/, '')
  }
  suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '')
  prefix = prefix.replace(/[._ -]+$/, '')

  const name = suffix ? prefix + suffix : prefix
  // Only when the shared part carries most of the name (not just "The.").
  return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest
}

/**
 * Release of a RAR volume set, from what its headers list: the shared
 * top-level directory if there is one, otherwise the first entry. For
 * volumes after the first, the first entry is the file that continues from
 * the previous volume, so all volumes of a single-file archive agree.
 */
/**
 * Folder and file names that say nothing about what a release is: disc
 * structures (VIDEO_TS on every DVD), samples, subtitles, disc numbers.
 * Kept in step with SQL's is_readable_name().
 */
export const GENERIC_NAMES = /^(video_ts|audio_ts|bdmv|certificate|hvdvd_ts|samples?|subs?|subtitles?|proof|cover|covers|extras?|featurettes?|(cd|dvd|disc|disk)[ ._-]?\d{1,2})$/i

export function isGenericName(name: string): boolean {
  return GENERIC_NAMES.test(name.trim())
}

export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
  if (entries.length === 0) return null
  const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
  const nested = entries.some((e) => /[\\/]/.test(e.name))
  // A generic top folder (VIDEO_TS) is shared by unrelated archives: key on the first entry instead.
  if (nested && tops.size === 1 && !isGenericName([...tops][0])) {
    const [top] = tops
    return { key: hash('rar-dir', top), name: top, source: 'rar' }
  }
  const first = entries[0]
  const base = first.name.split(/[\\/]/).pop() ?? first.name
  return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' }
}