import { execFile } from 'node:child_process' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { promisify } from 'node:util' import db from '@adonisjs/lucid/services/db' import logger from '@adonisjs/core/services/logger' import IngestService from '#services/IngestService' import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index' import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index' /** Stored per file and release; a listing of thousands of files is not useful. */ const MAX_ENTRIES = 500 export interface ArchiveResult { info: RarInfo releaseId?: number releaseRenamed?: string grouped?: boolean } /** * Records what a RAR volume's headers say: the listing on the file, the * release it belongs to (from the shared directory or continued entry, when * the file has none yet), and for first volumes the contents and password * flag on the release. See CONCEPT.md (D19, D21). */ class ArchiveService { async record(file: any, binary: any, info: RarInfo): Promise { const { next: _next, ...listing } = info const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) } await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) }) let releaseId = file.release_id ?? binary?.release_id ?? null let grouped = false const identity = rarReleaseIdentity(info.entries) if (!releaseId && identity) { const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [ { release: identity, poster: binary?.poster ?? file.poster, weakIdentity: true, groups: binary?.groups ?? file.groups ?? [], postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined, }, ]) releaseId = ids.get(identity.key)! grouped = true if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId }) await db.from('files').where('id', file.id).update({ release_id: releaseId }) } if (!releaseId) return { info } const release = await db.from('releases').where('id', releaseId).first() const update: Record = {} if (info.firstVolume !== false) { update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted) if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries) } else if (info.headersEncrypted) { update.encrypted = true } const better = releaseNameFromContents(info) // A spot's title is kept (CONCEPT.md D29). const renamed = release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined if (renamed) update.name = renamed if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update) if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`) return { info, releaseId, releaseRenamed: renamed, grouped } } } const run = promisify(execFile) /** Tools that print one archive entry to stdout, tried in order. */ const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [ ['bsdtar', (archive, entry) => ['-xOf', archive, entry]], ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]], ] /** * The unpacked data of one entry of a complete (single-volume) archive: * copied out when stored, else unpacked with bsdtar (libarchive, in the * Docker image) or unrar. Null when that is not possible. */ export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise { const stored = storedEntryData(archive, entry) if (stored) return stored if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null if (entry.size !== null && entry.size > maxBytes) return null const dir = await mkdtemp(path.join(tmpdir(), 'rar-')) try { const file = path.join(dir, 'archive.rar') await writeFile(file, archive) for (const [tool, args] of EXTRACTORS) { try { const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 }) if (stdout.length > 0) return stdout } catch (error: any) { if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`) } } return null } finally { await rm(dir, { recursive: true, force: true }) } } /** * A release name from an archive listing: the top-level directory shared by * all entries when it reads like a release name, otherwise the stem of the * largest entry with a readable name (e.g. an archive holding just * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped. */ export function releaseNameFromContents(info: Pick): string | undefined { const entries = info.entries.filter((e) => !e.directory) const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0])) if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) { const [top] = tops if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top } const candidates = entries .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 })) .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem)) .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem)) .sort((a, b) => b.size - a.size) return candidates[0]?.stem } /** * Renames random-named releases from their stored archive listing (no * downloads). Used by `release:rename`. */ export async function renameFromStoredContents(): Promise { const releases = await db .from('releases') .whereNotNull('contents') .whereRaw("jsonb_array_length(contents) > 0") .select('id', 'name', 'source', 'contents') let renamed = 0 for (const release of releases) { if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue const better = releaseNameFromContents({ entries: release.contents }) if (!better || better === release.name) continue await db.from('releases').where('id', release.id).update({ name: better }) renamed++ } return renamed } /** * Obfuscated names: one long token of mixed letters and digits, no * separators, and the other generated shapes seen on Usenet (LOG.md * 2026-10-01 (3)). Kept in step with SQL's is_readable_name(). */ export function looksRandom(name: string): boolean { // An unknown short extension (e.g. ".bin") says nothing about the name. name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '') // VIDEO_TS, Sample, CD1: real words, but no name. if (isGenericName(name)) return true if (/^[0-9a-f]{16,}$/i.test(name)) return true // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time) if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true // UUIDs if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true // 1650777167 (yEncBin), "83111763 n" (rclone uploads) if (/^\d{6,}( n)?$/.test(name)) return true // prvjr7hapd0i1ny14pz4, and rclone's long base32 names if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk). return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6 } export default new ArchiveService()