212 lines
8.2 KiB
TypeScript
212 lines
8.2 KiB
TypeScript
import { createHash } from 'node:crypto'
|
||
import { looksLikeFilename } from '../subject/file_key.js'
|
||
|
||
export interface ReleaseIdentity {
|
||
key: string
|
||
/** Human-readable name: the subject text or the shared filename stem. */
|
||
name: string
|
||
/** Number of files announced by the `[i/M]` counter, when present. */
|
||
fileTotal?: number
|
||
/** Which rule recognised the release. */
|
||
source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot'
|
||
}
|
||
|
||
export interface ReleaseInput {
|
||
poster: string
|
||
/** Subject without segment counter (the binary's `subject`). */
|
||
subject: string
|
||
/** Filename of the binary. */
|
||
name: string
|
||
/** True when the identity came from the yEnc body (obfuscated post). */
|
||
weak: boolean
|
||
}
|
||
|
||
// Suffixes that differ between the files of one release. Stripped repeatedly,
|
||
// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x".
|
||
const SUFFIXES = [
|
||
/\.vol\d+[+-]\d+\.par2$/i,
|
||
/\.par2$/i,
|
||
/\.part\d+\.rar$/i,
|
||
/\.r\d{2,3}$/i,
|
||
/\.rar$/i,
|
||
/\.7z(\.\d{3})?$/i,
|
||
/\.zip(\.\d{3})?$/i,
|
||
// Split volumes (x.mkv.001), not codec names (H.264, H.265).
|
||
/(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i,
|
||
/\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i,
|
||
/\.tar$/i,
|
||
/\.(zst|gz|xz|bz2)$/i,
|
||
/\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i,
|
||
]
|
||
|
||
/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */
|
||
export function fileStem(name: string): string {
|
||
let stem = name.trim()
|
||
let changed = true
|
||
while (changed) {
|
||
changed = false
|
||
for (const suffix of SUFFIXES) {
|
||
const next = stem.replace(suffix, '')
|
||
if (next !== stem && next.length > 0) {
|
||
stem = next
|
||
changed = true
|
||
}
|
||
}
|
||
}
|
||
return stem
|
||
}
|
||
|
||
/**
|
||
* What remains of a subject once the filename, counters, "yEnc" and sizes are
|
||
* removed. Posters that name the release outside the quotes put it here.
|
||
*/
|
||
export function subjectRemainder(subject: string): string {
|
||
return subject
|
||
.replace(/"[^"]*"/g, ' ')
|
||
.replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ')
|
||
// File sizes follow "yEnc" at the end; other numbers (years, post ids) stay.
|
||
.replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ')
|
||
.replace(/\byEnc\b/gi, ' ')
|
||
.replace(/[\s\-–.:|]+/g, ' ')
|
||
.trim()
|
||
}
|
||
|
||
/**
|
||
* Posting tools put random per-file tokens and poster IDs in subjects (e.g.
|
||
* `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a
|
||
* name once those are removed identifies a release.
|
||
*/
|
||
function isRandomToken(token: string): boolean {
|
||
const t = token.replace(/^\[|\]$/g, '')
|
||
if (t.length < 6) return false
|
||
if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true
|
||
return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t)
|
||
}
|
||
|
||
/**
|
||
* A stable per-post token: one short run of letters/digits with at least one
|
||
* digit. Requiring a digit keeps constant labels like `[PRiVATE]` from
|
||
* grouping different posts of the same poster.
|
||
*/
|
||
export function isPostToken(remainder: string): boolean {
|
||
const compact = remainder.replace(/[\s\[\]()_-]+/g, '')
|
||
return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact)
|
||
}
|
||
|
||
export function namesRelease(remainder: string): boolean {
|
||
const words = remainder
|
||
.split(/\s+/)
|
||
.filter((w) => w && !isRandomToken(w))
|
||
.join(' ')
|
||
// At least two words of letters, joined the way release names are.
|
||
return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words)
|
||
}
|
||
|
||
function hash(...parts: (string | number | undefined)[]) {
|
||
return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex')
|
||
}
|
||
|
||
/**
|
||
* Decides which release (download) a file belongs to. Returns null when the
|
||
* file cannot be grouped from its name alone (random extensionless names in
|
||
* obfuscated posts; those need par2 matching). See CONCEPT.md (D16).
|
||
*/
|
||
export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
||
const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject)
|
||
const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined
|
||
const stem = fileStem(input.name)
|
||
|
||
if (input.weak) {
|
||
// Obfuscating posters randomise the poster per article; a real filename
|
||
// stem is specific enough on its own.
|
||
if (!looksLikeFilename(input.name)) return null
|
||
return { key: hash('weak', stem), name: stem, source: 'weak-stem' }
|
||
}
|
||
|
||
// Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"".
|
||
const remainder = subjectRemainder(input.subject)
|
||
if (namesRelease(remainder)) {
|
||
return {
|
||
key: hash('subject', input.poster, remainder),
|
||
name: remainder,
|
||
fileTotal,
|
||
source: 'subject',
|
||
}
|
||
}
|
||
// Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the
|
||
// [i/M] counter it groups the files of one post whose names are random.
|
||
if (fileTotal && isPostToken(remainder)) {
|
||
return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' }
|
||
}
|
||
// The [i/M] total is informational, not part of the key: posters often
|
||
// upload the par2 index without a counter, and it belongs to the same post.
|
||
return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' }
|
||
}
|
||
|
||
/** Release of a par2 recovery set; every file listed in it belongs to it. */
|
||
export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity {
|
||
return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' }
|
||
}
|
||
|
||
const SEPARATOR = /[._ -]/
|
||
|
||
/**
|
||
* A release name from the files a par2 set lists. Files that differ only in
|
||
* the middle (episodes S03E01…S03E20, parts) give their shared name, with
|
||
* the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to
|
||
* SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato.
|
||
* Otherwise the longest stem, which tends to be the most descriptive (the
|
||
* video file).
|
||
*/
|
||
export function par2ReleaseName(fileNames: string[]): string | undefined {
|
||
const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean)
|
||
if (stems.length === 0) return undefined
|
||
const longest = [...stems].sort((a, b) => b.length - a.length)[0]
|
||
if (stems.length < 2) return longest
|
||
|
||
let prefix = stems.reduce((p, s) => {
|
||
let i = 0
|
||
while (i < p.length && i < s.length && p[i] === s[i]) i++
|
||
return p.slice(0, i)
|
||
})
|
||
let suffix = stems.reduce((p, s) => {
|
||
let i = 0
|
||
while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++
|
||
return p.slice(p.length - i)
|
||
})
|
||
const shortest = Math.min(...stems.map((s) => s.length))
|
||
if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest)
|
||
|
||
// Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token.
|
||
const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix)
|
||
if (season) prefix = season[1]
|
||
else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) {
|
||
prefix = prefix.replace(/[^._ -]*$/, '')
|
||
}
|
||
suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '')
|
||
prefix = prefix.replace(/[._ -]+$/, '')
|
||
|
||
const name = suffix ? prefix + suffix : prefix
|
||
// Only when the shared part carries most of the name (not just "The.").
|
||
return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest
|
||
}
|
||
|
||
/**
|
||
* Release of a RAR volume set, from what its headers list: the shared
|
||
* top-level directory if there is one, otherwise the first entry. For
|
||
* volumes after the first, the first entry is the file that continues from
|
||
* the previous volume, so all volumes of a single-file archive agree.
|
||
*/
|
||
export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null {
|
||
if (entries.length === 0) return null
|
||
const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0]))
|
||
const nested = entries.some((e) => /[\\/]/.test(e.name))
|
||
if (nested && tops.size === 1) {
|
||
const [top] = tops
|
||
return { key: hash('rar-dir', top), name: top, source: 'rar' }
|
||
}
|
||
const first = entries[0]
|
||
const base = first.name.split(/[\\/]/).pop() ?? first.name
|
||
return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' }
|
||
}
|