implemented a lot of features

2026-09-29 00:32:01 +02:00
1 parent 62f1767df3
commit bcb9be5da9
@@ -0,0 +1,4 @@
+/**
+ * Grouping of files into releases (downloads), independent of AdonisJS.
+ */
+export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js'
@@ -0,0 +1,211 @@
+import { createHash } from 'node:crypto'
+import { looksLikeFilename } from '../subject/file_key.js'
+
+export interface ReleaseIdentity {
+  key: string
+  /** Human-readable name: the subject text or the shared filename stem. */
+  name: string
+  /** Number of files announced by the `[i/M]` counter, when present. */
+  fileTotal?: number
+  /** Which rule recognised the release. */
+  source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot'
+}
+
+export interface ReleaseInput {
+  poster: string
+  /** Subject without segment counter (the binary's `subject`). */
+  subject: string
+  /** Filename of the binary. */
+  name: string
+  /** True when the identity came from the yEnc body (obfuscated post). */
+  weak: boolean
+}
+
+// Suffixes that differ between the files of one release. Stripped repeatedly,
+// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x".
+const SUFFIXES = [
+  /\.vol\d+[+-]\d+\.par2$/i,
+  /\.par2$/i,
+  /\.part\d+\.rar$/i,
+  /\.r\d{2,3}$/i,
+  /\.rar$/i,
+  /\.7z(\.\d{3})?$/i,
+  /\.zip(\.\d{3})?$/i,
+  // Split volumes (x.mkv.001), not codec names (H.264, H.265).
+  /(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i,
+  /\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
+  /\.tar$/i,
+  /\.(zst|gz|xz|bz2)$/i,
+  /\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
+]
+
+/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
+export function fileStem(name: string): string {
+  let stem = name.trim()
+  let changed = true
+  while (changed) {
+    changed = false
+    for (const suffix of SUFFIXES) {
+      const next = stem.replace(suffix, '')
+      if (next !== stem && next.length > 0) {
+        stem = next
+        changed = true
+      }
+    }
+  }
+  return stem
+}
+
+/**
+ * What remains of a subject once the filename, counters, "yEnc" and sizes are
+ * removed. Posters that name the release outside the quotes put it here.
+ */
+export function subjectRemainder(subject: string): string {
+  return subject
+    .replace(/"[^"]*"/g, ' ')
+    .replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
+    // File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
+    .replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
+    .replace(/\byEnc\b/gi, ' ')
+    .replace(/[\s\-–.:|]+/g, ' ')
+    .trim()
+}
+
+/**
+ * Posting tools put random per-file tokens and poster IDs in subjects (e.g.
+ * `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
+ * name once those are removed identifies a release.
+ */
+function isRandomToken(token: string): boolean {
+  const t = token.replace(/^\[|\]$/g, '')
+  if (t.length < 6) return false
+  if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true
+  return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t)
+}
+
+/**
+ * A stable per-post token: one short run of letters/digits with at least one
+ * digit. Requiring a digit keeps constant labels like `[PRiVATE]` from
+ * grouping different posts of the same poster.
+ */
+export function isPostToken(remainder: string): boolean {
+  const compact = remainder.replace(/[\s\[\]()_-]+/g, '')
+  return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact)
+}
+
+export function namesRelease(remainder: string): boolean {
+  const words = remainder
+    .split(/\s+/)
+    .filter((w) => w && !isRandomToken(w))
+    .join(' ')
+  // At least two words of letters, joined the way release names are.
+  return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words)
+}
+
+function hash(...parts: (string | number | undefined)[]) {
+  return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex')
+}
+
+/**
+ * Decides which release (download) a file belongs to. Returns null when the
+ * file cannot be grouped from its name alone (random extensionless names in
+ * obfuscated posts; those need par2 matching). See CONCEPT.md (D16).
+ */
+export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
+  const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
+  const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
+  const stem = fileStem(input.name)
+
+  if (input.weak) {
+    // Obfuscating posters randomise the poster per article; a real filename
+    // stem is specific enough on its own.
+    if (!looksLikeFilename(input.name)) return null
+    return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
+  }
+
+  // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
+  const remainder = subjectRemainder(input.subject)
+  if (namesRelease(remainder)) {
+    return {
+      key: hash('subject', input.poster, remainder),
+      name: remainder,
+      fileTotal,
+      source: 'subject',
+    }
+  }
+  // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
+  // [i/M] counter it groups the files of one post whose names are random.
+  if (fileTotal && isPostToken(remainder)) {
+    return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
+  }
+  // The [i/M] total is informational, not part of the key: posters often
+  // upload the par2 index without a counter, and it belongs to the same post.
+  return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
+}
+
+/** Release of a par2 recovery set; every file listed in it belongs to it. */
+export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity {
+  return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' }
+}
+
+const SEPARATOR = /[._ -]/
+
+/**
+ * A release name from the files a par2 set lists. Files that differ only in
+ * the middle (episodes S03E01…S03E20, parts) give their shared name, with
+ * the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to
+ * SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato.
+ * Otherwise the longest stem, which tends to be the most descriptive (the
+ * video file).
+ */
+export function par2ReleaseName(fileNames: string[]): string | undefined {
+  const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean)
+  if (stems.length === 0) return undefined
+  const longest = [...stems].sort((a, b) => b.length - a.length)[0]
+  if (stems.length < 2) return longest
+
+  let prefix = stems.reduce((p, s) => {
+    let i = 0
+    while (i < p.length && i < s.length && p[i] === s[i]) i++
+    return p.slice(0, i)
+  })
+  let suffix = stems.reduce((p, s) => {
+    let i = 0
+    while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++
+    return p.slice(p.length - i)
+  })
+  const shortest = Math.min(...stems.map((s) => s.length))
+  if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest)
+
+  // Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token.
+  const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix)
+  if (season) prefix = season[1]
+  else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) {
+    prefix = prefix.replace(/[^._ -]*$/, '')
+  }
+  suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '')
+  prefix = prefix.replace(/[._ -]+$/, '')
+
+  const name = suffix ? prefix + suffix : prefix
+  // Only when the shared part carries most of the name (not just "The.").
+  return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest
+}
+
+/**
+ * Release of a RAR volume set, from what its headers list: the shared
+ * top-level directory if there is one, otherwise the first entry. For
+ * volumes after the first, the first entry is the file that continues from
+ * the previous volume, so all volumes of a single-file archive agree.
+ */
+export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
+  if (entries.length === 0) return null
+  const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
+  const nested = entries.some((e) => /[\\/]/.test(e.name))
+  if (nested && tops.size === 1) {
+    const [top] = tops
+    return { key: hash('rar-dir', top), name: top, source: 'rar' }
+  }
+  const first = entries[0]
+  const base = first.name.split(/[\\/]/).pop() ?? first.name
+  return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' }
+}